From aaaeb84ac62db4b82721129c393f8ad9ffa67128 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 26 Aug 2016 10:02:10 -0700 Subject: [PATCH 01/16] Check for non-nil container after match There can be a race between getting the container ids for matches and getting the actual container. This makes sure that we check that the container returned by `Get` is non-nil before adding it to the list of matches. Fixes #25991 Signed-off-by: Michael Crosby (cherry picked from commit a020ec4c8b476a814eb137e216fe9d723524fc3b) Signed-off-by: Victor Vieux --- daemon/list.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon/list.go b/daemon/list.go index 6d86cd3b48..e55a38aa04 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -157,7 +157,9 @@ func (daemon *Daemon) filterByNameIDMatches(ctx *listContext) []*container.Conta cntrs := make([]*container.Container, 0, len(matches)) for id := range matches { - cntrs = append(cntrs, daemon.containers.Get(id)) + if c := daemon.containers.Get(id); c != nil { + cntrs = append(cntrs, c) + } } // Restore sort-order after filtering From 132a8e79634f7eee0db2ee19b0be9cf496f45076 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 5 Jul 2016 11:54:19 -0400 Subject: [PATCH 02/16] ppc64le: remove go SHA check Makes it consistent with other Dockerfiles. Signed-off-by: Christopher Jones (cherry picked from commit d9e12cba5a9a2291f56eb3921b06ac1b5f85dfc7) Signed-off-by: Victor Vieux --- Dockerfile.ppc64le | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile.ppc64le b/Dockerfile.ppc64le index 8527d1a54e..294d11d41d 100644 --- a/Dockerfile.ppc64le +++ b/Dockerfile.ppc64le @@ -91,11 +91,9 @@ RUN set -x \ # NOTE: ppc64le has compatibility issues with older versions of go, so make sure the version >= 1.6 ENV GO_VERSION 1.6.3 ENV GO_DOWNLOAD_URL https://golang.org/dl/go${GO_VERSION}.src.tar.gz -ENV GO_DOWNLOAD_SHA256 787b0b750d037016a30c6ed05a8a70a91b2e9db4bd9b1a2453aa502a63f1bccc ENV GOROOT_BOOTSTRAP /usr/local RUN curl -fsSL "$GO_DOWNLOAD_URL" -o golang.tar.gz \ - && echo "$GO_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ && tar -C /usr/src -xzf golang.tar.gz \ && rm golang.tar.gz \ && cd /usr/src/go/src && ./make.bash 2>&1 From fd7e763e7cc7c08133bbdd3af0fc864ea65e3804 Mon Sep 17 00:00:00 2001 From: Liam Macgillavry Date: Fri, 19 Aug 2016 10:43:15 +0200 Subject: [PATCH 03/16] missed DOCKER->DOCKERD change in 1ac1b78b3a771c562d9cfa91c14f8a494c3723c1 for 'status' Signed-off-by: Liam Macgillavry (cherry picked from commit 11eda60848e476c3a1d84a035642edba74e94c8a) Signed-off-by: Victor Vieux --- contrib/init/sysvinit-debian/docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/init/sysvinit-debian/docker b/contrib/init/sysvinit-debian/docker index c6a86d3401..30d14f1c42 100755 --- a/contrib/init/sysvinit-debian/docker +++ b/contrib/init/sysvinit-debian/docker @@ -139,7 +139,7 @@ case "$1" in status) check_init - status_of_proc -p "$DOCKER_SSD_PIDFILE" "$DOCKER" "$DOCKER_DESC" + status_of_proc -p "$DOCKER_SSD_PIDFILE" "$DOCKERD" "$DOCKER_DESC" ;; *) From 24df98212863a8a59de6e5d4e7759a75f706c3da Mon Sep 17 00:00:00 2001 From: Misty Stanley-Jones Date: Thu, 1 Sep 2016 15:38:25 -0700 Subject: [PATCH 04/16] Clarify usage of --force when used on a swarm manager Fixes #26125 Signed-off-by: Misty Stanley-Jones (cherry picked from commit 7b5c3d935a7a99b282f0f859101c887894a0b00e) Signed-off-by: Victor Vieux --- api/client/swarm/leave.go | 4 +-- docs/reference/commandline/node_rm.md | 37 ++++++++++++++--------- docs/reference/commandline/swarm_leave.md | 30 +++++++++--------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/api/client/swarm/leave.go b/api/client/swarm/leave.go index acaa7c82c1..52d83dcee6 100644 --- a/api/client/swarm/leave.go +++ b/api/client/swarm/leave.go @@ -19,7 +19,7 @@ func newLeaveCommand(dockerCli *client.DockerCli) *cobra.Command { cmd := &cobra.Command{ Use: "leave [OPTIONS]", - Short: "Leave a swarm", + Short: "Leave the swarm (workers only)", Args: cli.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runLeave(dockerCli, opts) @@ -27,7 +27,7 @@ func newLeaveCommand(dockerCli *client.DockerCli) *cobra.Command { } flags := cmd.Flags() - flags.BoolVar(&opts.force, "force", false, "Force leave ignoring warnings.") + flags.BoolVar(&opts.force, "force", false, "Force this node to leave the swarm, ignoring warnings") return cmd } diff --git a/docs/reference/commandline/node_rm.md b/docs/reference/commandline/node_rm.md index 6f1d13cd2b..36ff355ab9 100644 --- a/docs/reference/commandline/node_rm.md +++ b/docs/reference/commandline/node_rm.md @@ -25,36 +25,45 @@ Options: --help Print usage ``` -Removes specified nodes from a swarm. +When run from a manager node, removes the specified nodes from a swarm. Example output: - $ docker node rm swarm-node-02 - Node swarm-node-02 removed from swarm +```nohighlight +$ docker node rm swarm-node-02 -Removes nodes from the swarm that are in the down state. Attempting to remove -an active node will result in an error: - -```bash -$ docker node rm swarm-node-03 -Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not down and can't be removed +Node swarm-node-02 removed from swarm ``` -If a worker node becomes compromised, exhibits unexpected or unwanted behavior, or if you lose access to it so -that a clean shutdown is impossible, you can use the force option. +Removes the specified nodes from the swarm, but only if the nodes are in the +down state. If you attempt to remove an active node you will receive an error: -```bash +```nohighlight +$ docker node rm swarm-node-03 + +Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not +down and can't be removed +``` + +If you lose access to a worker node or need to shut it down because it has been +compromised or is not behaving as expected, you can use the `--force` option. +This may cause transient errors or interruptions, depending on the type of task +being run on the node. + +```nohighlight $ docker node rm --force swarm-node-03 + Node swarm-node-03 removed from swarm ``` -Note that manager nodes have to be demoted to worker nodes before they can be removed -from the cluster. +A manager node must be demoted to a worker node (using `docker node demote`) +before you can remove it from the swarm. ## Related information * [node inspect](node_inspect.md) * [node update](node_update.md) +* [node demote](node_demote.md) * [node ps](node_ps.md) * [node ls](node_ls.md) diff --git a/docs/reference/commandline/swarm_leave.md b/docs/reference/commandline/swarm_leave.md index d1de2306b8..ecdfc321f8 100644 --- a/docs/reference/commandline/swarm_leave.md +++ b/docs/reference/commandline/swarm_leave.md @@ -15,41 +15,41 @@ parent = "smn_cli" ```markdown Usage: docker swarm leave [OPTIONS] -Leave a swarm +Leave the swarm (workers only). Options: - --force Force leave ignoring warnings. + --force Force this node to leave the swarm, ignoring warnings --help Print usage ``` -This command causes the node to leave the swarm. +When you run this command on a worker, that worker leaves the swarm. -On a manager node: +You can use the `--force` option to on a manager to remove it from the swarm. +However, this does not reconfigure the swarm to ensure that there are enough +managers to maintain a quorum in the swarm. The safe way to remove a manager +from a swarm is to demote it to a worker and then direct it to leave the quorum +without using `--force`. Only use `--force` in situations where the swarm will +no longer be used after the manager leaves, such as in a single-node swarm. + +Consider the following swarm, as seen from the manager: ```bash $ docker node ls ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS 7ln70fl22uw2dvjn2ft53m3q5 worker2 Ready Active -dkp8vy1dq1kxleu9g4u78tlag worker1 Ready Active Reachable +dkp8vy1dq1kxleu9g4u78tlag worker1 Ready Active dvfxp4zseq4s0rih1selh0d20 * manager1 Ready Active Leader ``` -On a worker node, worker2 in the following example: +To remove `worker2`, issue the following command from `worker2` itself: ```bash $ docker swarm leave Node left the default swarm. ``` - -On a manager node: -```bash -$ docker node ls -ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS -7ln70fl22uw2dvjn2ft53m3q5 worker2 Down Active -dkp8vy1dq1kxleu9g4u78tlag worker1 Ready Active Reachable -dvfxp4zseq4s0rih1selh0d20 * manager1 Ready Active Leader -``` +To remove an inactive node, use the [`node rm`](swarm_rm.md) command instead. ## Related information +* [node rm](node_rm.md) * [swarm init](swarm_init.md) * [swarm join](swarm_join.md) * [swarm update](swarm_update.md) From e892a54fa13f8c634fe80e5b32ae783a5da03123 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 5 Sep 2016 03:11:14 -0700 Subject: [PATCH 05/16] Fix incorrect help output in `docker network ls` As is raised in 26312, in `docker network ls`, the help output was mistaken to `volume names`: ``` -q, --quiet Only display volume names ``` This fix changes the help output to: ``` -q, --quiet Only display network IDs ``` This fix also updates the documentation in: `docs/reference/commandline/network_ls.md` This fix fixes 26312. Signed-off-by: Yong Tang (cherry picked from commit b9e46235fadc6b390e2c04c44b6a865e4ea97cb8) Signed-off-by: Victor Vieux --- api/client/network/list.go | 2 +- docs/reference/commandline/network_ls.md | 2 +- man/docker-network-ls.1.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client/network/list.go b/api/client/network/list.go index c149b985c9..71c8f1a0b6 100644 --- a/api/client/network/list.go +++ b/api/client/network/list.go @@ -41,7 +41,7 @@ func newListCommand(dockerCli *client.DockerCli) *cobra.Command { } flags := cmd.Flags() - flags.BoolVarP(&opts.quiet, "quiet", "q", false, "Only display volume names") + flags.BoolVarP(&opts.quiet, "quiet", "q", false, "Only display network IDs") flags.BoolVar(&opts.noTrunc, "no-trunc", false, "Do not truncate the output") flags.StringSliceVarP(&opts.filter, "filter", "f", []string{}, "Provide filter values (i.e. 'dangling=true')") diff --git a/docs/reference/commandline/network_ls.md b/docs/reference/commandline/network_ls.md index 494fd3a922..3b321470e0 100644 --- a/docs/reference/commandline/network_ls.md +++ b/docs/reference/commandline/network_ls.md @@ -22,7 +22,7 @@ Options: -f, --filter value Provide filter values (i.e. 'dangling=true') (default []) --help Print usage --no-trunc Do not truncate the output - -q, --quiet Only display volume names + -q, --quiet Only display network IDs ``` Lists all the networks the Engine `daemon` knows about. This includes the diff --git a/man/docker-network-ls.1.md b/man/docker-network-ls.1.md index 3eeff05993..bfd87329cc 100644 --- a/man/docker-network-ls.1.md +++ b/man/docker-network-ls.1.md @@ -166,7 +166,7 @@ attached. Do not truncate the output **-q**, **--quiet**=*true*|*false* - Only display numeric IDs + Only display network IDs **--help** Print usage statement From 0dcd2e1a5d0258a4d83b5001c860614c26a1519d Mon Sep 17 00:00:00 2001 From: Yanqiang Miao Date: Tue, 20 Sep 2016 18:38:56 +0800 Subject: [PATCH 06/16] Remove the support of setting host configuration options when the container starts Signed-off-by: Yanqiang Miao update Signed-off-by: Yanqiang Miao update Signed-off-by: Yanqiang Miao update Signed-off-by: Yanqiang Miao (cherry picked from commit 26b6b47420be3e0d932a5e128ac2304b9e93efc2) Signed-off-by: Victor Vieux --- daemon/start.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/start.go b/daemon/start.go index 7a0bc2121c..fcf24c59f6 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -38,7 +38,7 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *containertypes.Hos // This is kept for backward compatibility - hostconfig should be passed when // creating a container, not during start. if hostConfig != nil { - logrus.Warn("DEPRECATED: Setting host configuration options when the container starts is deprecated and will be removed in Docker 1.12") + logrus.Warn("DEPRECATED: Setting host configuration options when the container starts is deprecated and has been removed in Docker 1.12") oldNetworkMode := container.HostConfig.NetworkMode if err := daemon.setSecurityOptions(container, hostConfig); err != nil { return err @@ -52,7 +52,7 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *containertypes.Hos newNetworkMode := container.HostConfig.NetworkMode if string(oldNetworkMode) != string(newNetworkMode) { // if user has change the network mode on starting, clean up the - // old networks. It is a deprecated feature and will be removed in Docker 1.12 + // old networks. It is a deprecated feature and has been removed in Docker 1.12 container.NetworkSettings.Networks = nil if err := container.ToDisk(); err != nil { return err From 2be91b121baba427fc6e45b9361c853ab6dae4f8 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Thu, 15 Sep 2016 10:53:48 -0700 Subject: [PATCH 07/16] Vendoring boltdb @fff57c100 Signed-off-by: Alessandro Boch (cherry picked from commit 5cfbdceafe87df8bab85c0561a1483d3191eb775) Signed-off-by: Victor Vieux --- hack/vendor.sh | 2 +- vendor/src/github.com/boltdb/bolt/README.md | 29 ++++++++++------- vendor/src/github.com/boltdb/bolt/bolt_386.go | 3 ++ .../src/github.com/boltdb/bolt/bolt_amd64.go | 3 ++ vendor/src/github.com/boltdb/bolt/bolt_arm.go | 21 ++++++++++++ .../src/github.com/boltdb/bolt/bolt_arm64.go | 3 ++ .../github.com/boltdb/bolt/bolt_ppc64le.go | 3 ++ .../src/github.com/boltdb/bolt/bolt_s390x.go | 3 ++ vendor/src/github.com/boltdb/bolt/bucket.go | 32 ++++++++++++++++++- vendor/src/github.com/boltdb/bolt/freelist.go | 20 ++++++++---- vendor/src/github.com/boltdb/bolt/node.go | 5 +++ vendor/src/github.com/boltdb/bolt/page.go | 6 ++++ 12 files changed, 110 insertions(+), 20 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 5d5ecaba71..e2849cb8b2 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -83,7 +83,7 @@ clone git github.com/coreos/etcd v2.3.2 fix_rewritten_imports github.com/coreos/etcd clone git github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 clone git github.com/hashicorp/consul v0.5.2 -clone git github.com/boltdb/bolt v1.2.1 +clone git github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages diff --git a/vendor/src/github.com/boltdb/bolt/README.md b/vendor/src/github.com/boltdb/bolt/README.md index 3bff3cce4b..2a69d95e78 100644 --- a/vendor/src/github.com/boltdb/bolt/README.md +++ b/vendor/src/github.com/boltdb/bolt/README.md @@ -1,4 +1,4 @@ -Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg) +Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) ==== Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] @@ -15,11 +15,11 @@ and setting values. That's it. ## Project Status -Bolt is stable and the API is fixed. Full unit test coverage and randomized -black box testing are used to ensure database consistency and thread safety. -Bolt is currently in high-load production environments serving databases as -large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed -services every day. +Bolt is stable, the API is fixed, and the file format is fixed. Full unit +test coverage and randomized black box testing are used to ensure database +consistency and thread safety. Bolt is currently in high-load production +environments serving databases as large as 1TB. Many companies such as +Shopify and Heroku use Bolt-backed services every day. ## Table of Contents @@ -209,7 +209,7 @@ and then safely close your transaction if an error is returned. This is the recommended way to use Bolt transactions. However, sometimes you may want to manually start and end your transactions. -You can use the `Tx.Begin()` function directly but **please** be sure to close +You can use the `DB.Begin()` function directly but **please** be sure to close the transaction. ```go @@ -313,7 +313,7 @@ func (s *Store) CreateUser(u *User) error { // Generate ID for the user. // This returns an error only if the Tx is closed or not writeable. // That can't happen in an Update() call so I ignore the error check. - id, _ = b.NextSequence() + id, _ := b.NextSequence() u.ID = int(id) // Marshal user data into bytes. @@ -448,6 +448,10 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Please note that keys and values in `ForEach()` are only valid while +the transaction is open. If you need to use a key or value outside of +the transaction, you must use `copy()` to copy it to another byte +slice. ### Nested buckets @@ -557,7 +561,7 @@ if err != nil { Bolt is able to run on mobile devices by leveraging the binding feature of the [gomobile](https://github.com/golang/mobile) tool. Create a struct that will contain your database logic and a reference to a `*bolt.DB` with a initializing -contstructor that takes in a filepath where the database file will be stored. +constructor that takes in a filepath where the database file will be stored. Neither Android nor iOS require extra permissions or cleanup from using this method. ```go @@ -807,6 +811,7 @@ them via pull request. Below is a list of public, open source projects that use Bolt: +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. * [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. * [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. @@ -825,7 +830,6 @@ Below is a list of public, open source projects that use Bolt: * [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. * [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database. * [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. @@ -842,9 +846,12 @@ Below is a list of public, open source projects that use Bolt: * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. -* [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. * [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/src/github.com/boltdb/bolt/bolt_386.go b/vendor/src/github.com/boltdb/bolt/bolt_386.go index e659bfb91f..820d533c15 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_386.go @@ -5,3 +5,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_amd64.go b/vendor/src/github.com/boltdb/bolt/bolt_amd64.go index cca6b7eb70..98fafdb47d 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_amd64.go @@ -5,3 +5,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_arm.go b/vendor/src/github.com/boltdb/bolt/bolt_arm.go index e659bfb91f..7e5cb4b941 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_arm.go @@ -1,7 +1,28 @@ package bolt +import "unsafe" + // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned bool + +func init() { + // Simple check to see whether this arch handles unaligned load/stores + // correctly. + + // ARM9 and older devices require load/stores to be from/to aligned + // addresses. If not, the lower 2 bits are cleared and that address is + // read in a jumbled up order. + + // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html + + raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11} + val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2)) + + brokenUnaligned = val != 0x11222211 +} diff --git a/vendor/src/github.com/boltdb/bolt/bolt_arm64.go b/vendor/src/github.com/boltdb/bolt/bolt_arm64.go index 6d2309352e..b26d84f91b 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_arm64.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go index 8351e129f6..8c143bc5d1 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_ppc64le.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bolt_s390x.go b/vendor/src/github.com/boltdb/bolt/bolt_s390x.go index f4dd26bbba..d7c39af925 100644 --- a/vendor/src/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/src/github.com/boltdb/bolt/bolt_s390x.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/src/github.com/boltdb/bolt/bucket.go b/vendor/src/github.com/boltdb/bolt/bucket.go index d2f8c524e4..511ce72d33 100644 --- a/vendor/src/github.com/boltdb/bolt/bucket.go +++ b/vendor/src/github.com/boltdb/bolt/bucket.go @@ -130,9 +130,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket { func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) + // If unaligned load/stores are broken on this arch and value is + // unaligned simply clone to an aligned byte array. + unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0 + + if unaligned { + value = cloneBytes(value) + } + // If this is a writable transaction then we need to copy the bucket entry. // Read-only transactions can point directly at the mmap entry. - if b.tx.writable { + if b.tx.writable && !unaligned { child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) } else { @@ -329,6 +337,28 @@ func (b *Bucket) Delete(key []byte) error { return nil } +// Sequence returns the current integer for the bucket without incrementing it. +func (b *Bucket) Sequence() uint64 { return b.bucket.sequence } + +// SetSequence updates the sequence number for the bucket. +func (b *Bucket) SetSequence(v uint64) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Materialize the root node if it hasn't been already so that the + // bucket will be saved during commit. + if b.rootNode == nil { + _ = b.node(b.root, nil) + } + + // Increment and return the sequence. + b.bucket.sequence = v + return nil +} + // NextSequence returns an autoincrementing integer for the bucket. func (b *Bucket) NextSequence() (uint64, error) { if b.tx.db == nil { diff --git a/vendor/src/github.com/boltdb/bolt/freelist.go b/vendor/src/github.com/boltdb/bolt/freelist.go index 0161948fcf..d32f6cd937 100644 --- a/vendor/src/github.com/boltdb/bolt/freelist.go +++ b/vendor/src/github.com/boltdb/bolt/freelist.go @@ -166,12 +166,16 @@ func (f *freelist) read(p *page) { } // Copy the list of page ids from the freelist. - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + f.ids = make([]pgid, len(ids)) + copy(f.ids, ids) - // Make sure they're sorted. - sort.Sort(pgids(f.ids)) + // Make sure they're sorted. + sort.Sort(pgids(f.ids)) + } // Rebuild the page cache. f.reindex() @@ -189,7 +193,9 @@ func (f *freelist) write(p *page) error { // The page.count can only hold up to 64k elements so if we overflow that // number then we handle it by putting the size in the first element. - if len(ids) < 0xFFFF { + if len(ids) == 0 { + p.count = uint16(len(ids)) + } else if len(ids) < 0xFFFF { p.count = uint16(len(ids)) copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids) } else { @@ -230,7 +236,7 @@ func (f *freelist) reload(p *page) { // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { - f.cache = make(map[pgid]bool) + f.cache = make(map[pgid]bool, len(f.ids)) for _, id := range f.ids { f.cache[id] = true } diff --git a/vendor/src/github.com/boltdb/bolt/node.go b/vendor/src/github.com/boltdb/bolt/node.go index e9d64af81e..159318b229 100644 --- a/vendor/src/github.com/boltdb/bolt/node.go +++ b/vendor/src/github.com/boltdb/bolt/node.go @@ -201,6 +201,11 @@ func (n *node) write(p *page) { } p.count = uint16(len(n.inodes)) + // Stop here if there are no items to write. + if p.count == 0 { + return + } + // Loop over each item and write it to the page. b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] for i, item := range n.inodes { diff --git a/vendor/src/github.com/boltdb/bolt/page.go b/vendor/src/github.com/boltdb/bolt/page.go index 4a555286a3..7651a6bf7d 100644 --- a/vendor/src/github.com/boltdb/bolt/page.go +++ b/vendor/src/github.com/boltdb/bolt/page.go @@ -62,6 +62,9 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -72,6 +75,9 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] } From 708a93a8008d35ba659f42e67c476fb85fbffe48 Mon Sep 17 00:00:00 2001 From: Misty Stanley-Jones Date: Fri, 2 Sep 2016 10:21:55 -0700 Subject: [PATCH 08/16] 'docker node inspect --pretty' needs an extra newline at the end Signed-off-by: Misty Stanley-Jones (cherry picked from commit 341489f1500873a6aa51998f49c1da7f7fd8eb5a) Signed-off-by: Victor Vieux --- api/client/node/inspect.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client/node/inspect.go b/api/client/node/inspect.go index 835c784e05..6fcd78e932 100644 --- a/api/client/node/inspect.go +++ b/api/client/node/inspect.go @@ -71,6 +71,8 @@ func printHumanFriendly(out io.Writer, refs []string, getRef inspect.GetRefFunc) // print extra space between objects, but not after the last one if idx+1 != len(refs) { fmt.Fprintf(out, "\n\n") + } else { + fmt.Fprintf(out, "\n") } } return nil From a1d191d91ee60f69f5cce4b3bb0acff0d80a404b Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Sat, 17 Sep 2016 10:25:05 -0400 Subject: [PATCH 09/16] re-vendor syslog package Fixes #26394 Signed-off-by: Brian Goff (cherry picked from commit f528690674712b680caf2712092c7e2f8f236491) Signed-off-by: Victor Vieux --- hack/vendor.sh | 2 +- vendor/src/github.com/RackSec/srslog/formatter.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index e2849cb8b2..2859554d66 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -61,7 +61,7 @@ clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://gith clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d clone git github.com/docker/engine-api 4eca04ae18f4f93f40196a17b9aa6e11262a7269 -clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837 +clone git github.com/RackSec/srslog 365bf33cd9acc21ae1c355209865f17228ca534e clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages diff --git a/vendor/src/github.com/RackSec/srslog/formatter.go b/vendor/src/github.com/RackSec/srslog/formatter.go index 2a74625109..7852ad37e4 100644 --- a/vendor/src/github.com/RackSec/srslog/formatter.go +++ b/vendor/src/github.com/RackSec/srslog/formatter.go @@ -32,7 +32,7 @@ func UnixFormatter(p Priority, hostname, tag, content string) string { // RFC3164Formatter provides an RFC 3164 compliant message. func RFC3164Formatter(p Priority, hostname, tag, content string) string { timestamp := time.Now().Format(time.Stamp) - msg := fmt.Sprintf("<%d> %s %s %s[%d]: %s", + msg := fmt.Sprintf("<%d>%s %s %s[%d]: %s", p, timestamp, hostname, tag, os.Getpid(), content) return msg } From 5619e72a346b9ae274241bd2c0f512194052398e Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Tue, 13 Sep 2016 09:25:36 -0700 Subject: [PATCH 10/16] Add retry logic during aufs unmount. Treat EBUSY as a transient error and retry. Also stop ignoring unmount errors. Signed-off-by: Anusha Ragunathan (cherry picked from commit 0e539fec331cb9dbc4ef784b55516570b11affe2) Signed-off-by: Victor Vieux --- daemon/graphdriver/aufs/aufs.go | 73 +++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index d7177d3348..fac69b8740 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -33,6 +33,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/vbatts/tar-split/tar/storage" @@ -264,6 +265,39 @@ func (a *Driver) createDirsFor(id string) error { return nil } +// Helper function to debug EBUSY errors on remove. +func debugEBusy(mountPath string) (out []string, err error) { + // lsof is not part of GNU coreutils. This is a best effort + // attempt to detect offending processes. + c := exec.Command("lsof") + + r, err := c.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("Assigning pipes failed with %v", err) + } + + if err := c.Start(); err != nil { + return nil, fmt.Errorf("Starting %s failed with %v", c.Path, err) + } + + defer func() { + waiterr := c.Wait() + if waiterr != nil && err == nil { + err = fmt.Errorf("Waiting for %s failed with %v", c.Path, waiterr) + } + }() + + sc := bufio.NewScanner(r) + for sc.Scan() { + entry := sc.Text() + if strings.Contains(entry, mountPath) { + out = append(out, entry, "\n") + } + } + + return out, nil +} + // Remove will unmount and remove the given id. func (a *Driver) Remove(id string) error { a.pathCacheLock.Lock() @@ -272,9 +306,35 @@ func (a *Driver) Remove(id string) error { if !exists { mountpoint = a.getMountpoint(id) } - if err := a.unmount(mountpoint); err != nil { - // no need to return here, we can still try to remove since the `Rename` will fail below if still mounted - logrus.Debugf("aufs: error while unmounting %s: %v", mountpoint, err) + + var retries int + for { + mounted, err := a.mounted(mountpoint) + if err != nil { + return err + } + if !mounted { + break + } + + if err := a.unmount(mountpoint); err != nil { + if err != syscall.EBUSY { + return fmt.Errorf("aufs: unmount error: %s: %v", mountpoint, err) + } + if retries >= 5 { + out, debugErr := debugEBusy(mountpoint) + if debugErr == nil { + logrus.Warnf("debugEBusy returned %v", out) + } + return fmt.Errorf("aufs: unmount error after retries: %s: %v", mountpoint, err) + } + // If unmount returns EBUSY, it could be a transient error. Sleep and retry. + retries++ + logrus.Warnf("unmount failed due to EBUSY: retry count: %d", retries) + time.Sleep(100 * time.Millisecond) + continue + } + break } // Atomically remove each directory in turn by first moving it out of the @@ -282,6 +342,13 @@ func (a *Driver) Remove(id string) error { // the whole tree. tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id)) if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) { + if err == syscall.EBUSY { + logrus.Warnf("os.Rename err due to EBUSY") + out, debugErr := debugEBusy(mountpoint) + if debugErr == nil { + logrus.Warnf("debugEBusy returned %v", out) + } + } return err } defer os.RemoveAll(tmpMntPath) From f4650e2eb8fe3c1c43cb702002a92237809104a1 Mon Sep 17 00:00:00 2001 From: Ron Williams Date: Mon, 8 Aug 2016 20:59:52 -0700 Subject: [PATCH 11/16] Force input stream ANSI emulation for ConEmu. Signed-off-by: Ron Williams (cherry picked from commit 0fd4bbda2dfb9a1bfc6781f2a3dacedc62c7c11b) Signed-off-by: Victor Vieux --- pkg/term/term_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/term/term_windows.go b/pkg/term/term_windows.go index dc50da4577..11a16fdea8 100644 --- a/pkg/term/term_windows.go +++ b/pkg/term/term_windows.go @@ -73,6 +73,7 @@ func StdStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) { if os.Getenv("ConEmuANSI") == "ON" { // The ConEmu terminal emulates ANSI on output streams well. + emulateStdin = true emulateStdout = false emulateStderr = false } From 5592ee4d0748a99fd877c065ad72bb7ffc3f59b6 Mon Sep 17 00:00:00 2001 From: Thomas Leonard Date: Wed, 31 Aug 2016 17:36:10 +0100 Subject: [PATCH 12/16] Fix exec form of HEALTHCHECK CMD We attached the JSON flag to the wrong AST node, causing Docker to treat the exec form ["binary", "arg"] as if the shell form "binary arg" had been used. This failed if "ls" was not present. Added a test to detect this. Fixes #26174 Signed-off-by: Thomas Leonard (cherry picked from commit e95b6b51daed868094c7b66113381d5088e831b4) Signed-off-by: Victor Vieux --- builder/dockerfile/parser/line_parsers.go | 2 +- integration-cli/docker_cli_health_test.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/builder/dockerfile/parser/line_parsers.go b/builder/dockerfile/parser/line_parsers.go index dec7a757e8..d82b19a081 100644 --- a/builder/dockerfile/parser/line_parsers.go +++ b/builder/dockerfile/parser/line_parsers.go @@ -357,5 +357,5 @@ func parseHealthConfig(rest string, d *Directive) (*Node, map[string]bool, error return nil, nil, err } - return &Node{Value: typ, Next: cmd, Attributes: attrs}, nil, err + return &Node{Value: typ, Next: cmd}, attrs, err } diff --git a/integration-cli/docker_cli_health_test.go b/integration-cli/docker_cli_health_test.go index d231965332..28ef47dcc2 100644 --- a/integration-cli/docker_cli_health_test.go +++ b/integration-cli/docker_cli_health_test.go @@ -149,4 +149,19 @@ func (s *DockerSuite) TestHealth(c *check.C) { c.Check(last.ExitCode, checker.Equals, -1) c.Check(last.Output, checker.Equals, "Health check exceeded timeout (1ms)") dockerCmd(c, "rm", "-f", "test") + + // Check JSON-format + _, err = buildImage(imageName, + `FROM busybox + RUN echo OK > /status + CMD ["/bin/sleep", "120"] + STOPSIGNAL SIGKILL + HEALTHCHECK --interval=1s --timeout=30s \ + CMD ["cat", "/my status"]`, + true) + c.Check(err, check.IsNil) + out, _ = dockerCmd(c, "inspect", + "--format={{.Config.Healthcheck.Test}}", imageName) + c.Check(out, checker.Equals, "[CMD cat /my status]\n") + } From 32b0633f65fae131a1ed05e69878bbfb94d320cc Mon Sep 17 00:00:00 2001 From: Satoshi Tagomori Date: Tue, 20 Sep 2016 20:31:22 +0900 Subject: [PATCH 13/16] Update fluent-logger-golang to v1.2.0. Fix race condition issue to solve an issue about "panic: runtime error: invalid memory address or nil pointer dereference". This fix stabilize Docker daemon under the situation of communication problem with Fluentd processes. Signed-off-by: Satoshi Tagomori (cherry picked from commit 87124b9d62bc71f7632126cf7f8d5eb805c4a7f1) Signed-off-by: Victor Vieux --- hack/vendor.sh | 2 +- .../fluent/fluent-logger-golang/LICENSE | 209 +++++++++++++++++- .../fluent-logger-golang/fluent/fluent.go | 178 +++++++++------ 3 files changed, 309 insertions(+), 80 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 2859554d66..7a37064a7e 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -114,7 +114,7 @@ clone git github.com/golang/protobuf 3c84672111d91bb5ac31719e112f9f7126a0e26e # gelf logging driver deps clone git github.com/Graylog2/go-gelf aab2f594e4585d43468ac57287b0dece9d806883 -clone git github.com/fluent/fluent-logger-golang v1.1.0 +clone git github.com/fluent/fluent-logger-golang v1.2.0 # fluent-logger-golang deps clone git github.com/philhofer/fwd 899e4efba8eaa1fea74175308f3fae18ff3319fa clone git github.com/tinylib/msgp 75ee40d2601edf122ef667e2a07d600d4c44490c diff --git a/vendor/src/github.com/fluent/fluent-logger-golang/LICENSE b/vendor/src/github.com/fluent/fluent-logger-golang/LICENSE index 1aa91de6cc..d645695673 100644 --- a/vendor/src/github.com/fluent/fluent-logger-golang/LICENSE +++ b/vendor/src/github.com/fluent/fluent-logger-golang/LICENSE @@ -1,13 +1,202 @@ -Copyright (c) 2013 Tatsuo Kaniwa -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/src/github.com/fluent/fluent-logger-golang/fluent/fluent.go b/vendor/src/github.com/fluent/fluent-logger-golang/fluent/fluent.go index 1647e9e9e6..7363fa3f1e 100644 --- a/vendor/src/github.com/fluent/fluent-logger-golang/fluent/fluent.go +++ b/vendor/src/github.com/fluent/fluent-logger-golang/fluent/fluent.go @@ -1,6 +1,7 @@ package fluent import ( + "encoding/json" "errors" "fmt" "io" @@ -25,24 +26,28 @@ const ( ) type Config struct { - FluentPort int - FluentHost string - FluentNetwork string - FluentSocketPath string - Timeout time.Duration - BufferLimit int - RetryWait int - MaxRetry int - TagPrefix string - AsyncConnect bool + FluentPort int `json:"fluent_port"` + FluentHost string `json:"fluent_host"` + FluentNetwork string `json:"fluent_network"` + FluentSocketPath string `json:"fluent_socket_path"` + Timeout time.Duration `json:"timeout"` + BufferLimit int `json:"buffer_limit"` + RetryWait int `json:"retry_wait"` + MaxRetry int `json:"max_retry"` + TagPrefix string `json:"tag_prefix"` + AsyncConnect bool `json:"async_connect"` + MarshalAsJSON bool `json:"marshal_as_json"` } type Fluent struct { Config + + mubuff sync.Mutex + pending []byte + + muconn sync.Mutex conn io.WriteCloser - pending []byte reconnecting bool - mu sync.Mutex } // New creates a new Logger. @@ -140,9 +145,9 @@ func (f *Fluent) PostWithTime(tag string, tm time.Time, message interface{}) err } if msgtype.Kind() != reflect.Map { - return errors.New("messge must be a map") + return errors.New("fluent#PostWithTime: message must be a map") } else if msgtype.Key().Kind() != reflect.String { - return errors.New("map keys must be strings") + return errors.New("fluent#PostWithTime: map keys must be strings") } kv := make(map[string]interface{}) @@ -154,33 +159,54 @@ func (f *Fluent) PostWithTime(tag string, tm time.Time, message interface{}) err } func (f *Fluent) EncodeAndPostData(tag string, tm time.Time, message interface{}) error { - if data, dumperr := f.EncodeData(tag, tm, message); dumperr != nil { - return fmt.Errorf("fluent#EncodeAndPostData: can't convert '%s' to msgpack:%s", message, dumperr) - // fmt.Println("fluent#Post: can't convert to msgpack:", message, dumperr) - } else { - f.PostRawData(data) - return nil + var data []byte + var err error + if data, err = f.EncodeData(tag, tm, message); err != nil { + return fmt.Errorf("fluent#EncodeAndPostData: can't convert '%#v' to msgpack:%v", message, err) } + return f.postRawData(data) } +// Deprecated: Use EncodeAndPostData instead func (f *Fluent) PostRawData(data []byte) { - f.mu.Lock() - f.pending = append(f.pending, data...) - f.mu.Unlock() + f.postRawData(data) +} + +func (f *Fluent) postRawData(data []byte) error { + if err := f.appendBuffer(data); err != nil { + return err + } if err := f.send(); err != nil { f.close() - if len(f.pending) > f.Config.BufferLimit { - f.flushBuffer() - } - } else { - f.flushBuffer() + return err } + return nil +} + +// For sending forward protocol adopted JSON +type MessageChunk struct { + message Message +} + +// Golang default marshaler does not support +// ["value", "value2", {"key":"value"}] style marshaling. +// So, it should write JSON marshaler by hand. +func (chunk *MessageChunk) MarshalJSON() ([]byte, error) { + data, err := json.Marshal(chunk.message.Record) + return []byte(fmt.Sprintf("[\"%s\",%d,%s,null]", chunk.message.Tag, + chunk.message.Time, data)), err } func (f *Fluent) EncodeData(tag string, tm time.Time, message interface{}) (data []byte, err error) { timeUnix := tm.Unix() - msg := &Message{Tag: tag, Time: timeUnix, Record: message} - data, err = msg.MarshalMsg(nil) + if f.Config.MarshalAsJSON { + msg := Message{Tag: tag, Time: timeUnix, Record: message} + chunk := &MessageChunk{message: msg} + data, err = json.Marshal(chunk) + } else { + msg := &Message{Tag: tag, Time: timeUnix, Record: message} + data, err = msg.MarshalMsg(nil) + } return } @@ -193,23 +219,32 @@ func (f *Fluent) Close() (err error) { return } -// close closes the connection. -func (f *Fluent) close() (err error) { - if f.conn != nil { - f.mu.Lock() - defer f.mu.Unlock() - } else { - return +// appendBuffer appends data to buffer with lock. +func (f *Fluent) appendBuffer(data []byte) error { + f.mubuff.Lock() + defer f.mubuff.Unlock() + if len(f.pending)+len(data) > f.Config.BufferLimit { + return errors.New(fmt.Sprintf("fluent#appendBuffer: Buffer full, limit %v", f.Config.BufferLimit)) } + f.pending = append(f.pending, data...) + return nil +} + +// close closes the connection. +func (f *Fluent) close() { + f.muconn.Lock() if f.conn != nil { f.conn.Close() f.conn = nil } - return + f.muconn.Unlock() } // connect establishes a new connection using the specified transport. func (f *Fluent) connect() (err error) { + f.muconn.Lock() + defer f.muconn.Unlock() + switch f.Config.FluentNetwork { case "tcp": f.conn, err = net.DialTimeout(f.Config.FluentNetwork, f.Config.FluentHost+":"+strconv.Itoa(f.Config.FluentPort), f.Config.Timeout) @@ -218,6 +253,10 @@ func (f *Fluent) connect() (err error) { default: err = net.UnknownNetworkError(f.Config.FluentNetwork) } + + if err != nil { + f.reconnecting = false + } return } @@ -226,44 +265,45 @@ func e(x, y float64) int { } func (f *Fluent) reconnect() { - go func() { - for i := 0; ; i++ { - err := f.connect() - if err == nil { - f.mu.Lock() - f.reconnecting = false - f.mu.Unlock() - break - } else { - if i == f.Config.MaxRetry { - panic("fluent#reconnect: failed to reconnect!") - } - waitTime := f.Config.RetryWait * e(defaultReconnectWaitIncreRate, float64(i-1)) - time.Sleep(time.Duration(waitTime) * time.Millisecond) - } + for i := 0; ; i++ { + err := f.connect() + if err == nil { + f.send() + return } - }() + if i == f.Config.MaxRetry { + // TODO: What we can do when connection failed MaxRetry times? + panic("fluent#reconnect: failed to reconnect!") + } + waitTime := f.Config.RetryWait * e(defaultReconnectWaitIncreRate, float64(i-1)) + time.Sleep(time.Duration(waitTime) * time.Millisecond) + } } -func (f *Fluent) flushBuffer() { - f.mu.Lock() - defer f.mu.Unlock() - f.pending = f.pending[0:0] -} +func (f *Fluent) send() error { + f.muconn.Lock() + defer f.muconn.Unlock() -func (f *Fluent) send() (err error) { if f.conn == nil { if f.reconnecting == false { - f.mu.Lock() f.reconnecting = true - f.mu.Unlock() - f.reconnect() + go f.reconnect() } - err = errors.New("fluent#send: can't send logs, client is reconnecting") - } else { - f.mu.Lock() - _, err = f.conn.Write(f.pending) - f.mu.Unlock() + return errors.New("fluent#send: can't send logs, client is reconnecting") } - return + + f.mubuff.Lock() + defer f.mubuff.Unlock() + + var err error + if len(f.pending) > 0 { + _, err = f.conn.Write(f.pending) + if err != nil { + f.conn.Close() + f.conn = nil + } else { + f.pending = f.pending[:0] + } + } + return err } From f2a48d2ff3b6efab4148389d291072250c2628b8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 23 Aug 2016 21:08:23 -0700 Subject: [PATCH 14/16] Fix AuthZ plugins headers change issue This fix tries to address the issue raised in 25927 where the HTTP headers have been chaged when AUthZ plugin is in place. This issue is that in `FlushAll` (`pkg/authorization/response.go`), the headers have been written (with `WriteHeader`) before all the headers have bee copied. This fix fixes the issue by placing `WriteHeader` after. A test has been added to cover the changes.` This fix fixes 25927 Signed-off-by: Yong Tang (cherry picked from commit 9cb8fb6ea03fcd78010ce7dd33585d96cd73e38c) Signed-off-by: Victor Vieux --- integration-cli/docker_cli_authz_unix_test.go | 19 +++++++++++++++++++ pkg/authorization/response.go | 12 +++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/integration-cli/docker_cli_authz_unix_test.go b/integration-cli/docker_cli_authz_unix_test.go index 427319a89e..a78fac0619 100644 --- a/integration-cli/docker_cli_authz_unix_test.go +++ b/integration-cli/docker_cli_authz_unix_test.go @@ -443,6 +443,25 @@ func (s *DockerAuthzSuite) TestAuthZPluginEnsureLoadImportWorking(c *check.C) { c.Assert(err, check.IsNil, check.Commentf(out)) } +func (s *DockerAuthzSuite) TestAuthZPluginHeader(c *check.C) { + c.Assert(s.d.Start("--debug", "--authorization-plugin="+testAuthZPlugin), check.IsNil) + s.ctrl.reqRes.Allow = true + s.ctrl.resRes.Allow = true + c.Assert(s.d.LoadBusybox(), check.IsNil) + + daemonURL, err := url.Parse(s.d.sock()) + + conn, err := net.DialTimeout(daemonURL.Scheme, daemonURL.Path, time.Second*10) + c.Assert(err, check.IsNil) + client := httputil.NewClientConn(conn, nil) + req, err := http.NewRequest("GET", "/version", nil) + c.Assert(err, check.IsNil) + resp, err := client.Do(req) + + c.Assert(err, check.IsNil) + c.Assert(resp.Header["Content-Type"][0], checker.Equals, "application/json") +} + // assertURIRecorded verifies that the given URI was sent and recorded in the authz plugin func assertURIRecorded(c *check.C, uris []string, uri string) { var found bool diff --git a/pkg/authorization/response.go b/pkg/authorization/response.go index 9eec82d42e..f7ce7364ce 100644 --- a/pkg/authorization/response.go +++ b/pkg/authorization/response.go @@ -175,11 +175,6 @@ func (rm *responseModifier) Flush() { // FlushAll flushes all data to the HTTP response func (rm *responseModifier) FlushAll() error { - // Copy the status code - if rm.statusCode > 0 { - rm.rw.WriteHeader(rm.statusCode) - } - // Copy the header for k, vv := range rm.header { for _, v := range vv { @@ -187,6 +182,13 @@ func (rm *responseModifier) FlushAll() error { } } + // Copy the status code + // Also WriteHeader needs to be done after all the headers + // have been copied (above). + if rm.statusCode > 0 { + rm.rw.WriteHeader(rm.statusCode) + } + var err error if len(rm.body) > 0 { // Write body From d5aaaa7ea3ba0601d30be418e9aae681569f260e Mon Sep 17 00:00:00 2001 From: "Ji.Zhilong" Date: Fri, 15 Jul 2016 22:47:31 +0800 Subject: [PATCH 15/16] devmapper: prevent libdevmapper from deleting device symlinks in RemoveDeviceDeferred if there is no cookie set in dm task, or flag DM_UDEV_DISABLE_LIBRARY_FALLBACK is cleared for a DM_DEV_REMOVE task, libdevmapper will fallback to clean up the symlink under /dev/mapper by itself, no matter the device removal is executed immediately or deferred by the kernel.In some cases, the removal is deferred by the kernel, while the symlink is deleted directly by libdevmapper, when docker tries to activate the device again, the deferred removal will be canceld, but the symlink will not show up again, so docker's attempt to mount the device by the symlink will fail, and it will eventually leads to a `docker start/diff` error. Fixes #24671 Signed-off-by: Ji.Zhilong (cherry picked from commit 5e505d101f0201a4d045510d0a9b0c66697dedfe) Signed-off-by: Victor Vieux --- pkg/devicemapper/devmapper.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/devicemapper/devmapper.go b/pkg/devicemapper/devmapper.go index 838b06af40..724ef63bf4 100644 --- a/pkg/devicemapper/devmapper.go +++ b/pkg/devicemapper/devmapper.go @@ -358,6 +358,27 @@ func RemoveDeviceDeferred(name string) error { return ErrTaskDeferredRemove } + // set a task cookie and disable library fallback, or else libdevmapper will + // disable udev dm rules and delete the symlink under /dev/mapper by itself, + // even if the removal is deferred by the kernel. + var cookie uint + var flags uint16 + flags = DmUdevDisableLibraryFallback + if err := task.setCookie(&cookie, flags); err != nil { + return fmt.Errorf("devicemapper: Can not set cookie: %s", err) + } + + // libdevmapper and udev relies on System V semaphore for synchronization, + // semaphores created in `task.setCookie` will be cleaned up in `UdevWait`. + // So these two function call must come in pairs, otherwise semaphores will + // be leaked, and the limit of number of semaphores defined in `/proc/sys/kernel/sem` + // will be reached, which will eventually make all follwing calls to 'task.SetCookie' + // fail. + // this call will not wait for the deferred removal's final executing, since no + // udev event will be generated, and the semaphore's value will not be incremented + // by udev, what UdevWait is just cleaning up the semaphore. + defer UdevWait(&cookie) + if err = task.run(); err != nil { return fmt.Errorf("devicemapper: Error running RemoveDeviceDeferred %s", err) } From b6943c50f14324579916f47bb3c7820371faa4c1 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Wed, 31 Aug 2016 11:44:32 -0700 Subject: [PATCH 16/16] cluster: Do not autodetect advertise address on join On join, remote addresses are supposed to be detected by the manager that receives the join request. However, the daemon is interfering with this by automatically detecting an advertise address and specifying that to the remote manager. Fix this so that an advertise address is only specified while joining a cluster if one was given by the user. Signed-off-by: Aaron Lehmann (cherry picked from commit b1d2b088533187954d3b98ed5951ec2dbbb422e9) Signed-off-by: Victor Vieux --- daemon/cluster/cluster.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/daemon/cluster/cluster.go b/daemon/cluster/cluster.go index f09c5ee398..04a65f01f4 100644 --- a/daemon/cluster/cluster.go +++ b/daemon/cluster/cluster.go @@ -455,11 +455,13 @@ func (c *Cluster) Join(req types.JoinRequest) error { } var advertiseAddr string - advertiseHost, advertisePort, err := c.resolveAdvertiseAddr(req.AdvertiseAddr, listenPort) - // For joining, we don't need to provide an advertise address, - // since the remote side can detect it. - if err == nil { - advertiseAddr = net.JoinHostPort(advertiseHost, advertisePort) + if req.AdvertiseAddr != "" { + advertiseHost, advertisePort, err := c.resolveAdvertiseAddr(req.AdvertiseAddr, listenPort) + // For joining, we don't need to provide an advertise address, + // since the remote side can detect it. + if err == nil { + advertiseAddr = net.JoinHostPort(advertiseHost, advertisePort) + } } // todo: check current state existing