From d1d6357beb425500b517c708db645afa681d7136 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 16 Nov 2016 21:46:37 -0800 Subject: [PATCH 01/68] Convert DanglingOnly to Filters for `docker image prune` This fix convert DanglingOnly in ImagesPruneConfig to Filters, so that it is possible to maintain API compatibility in the future. Several integration tests have been added to cover changes. This fix is related to 28497. A follow up to this PR will be done once this PR is merged. Signed-off-by: Yong Tang (cherry picked from commit a6be56b54e871c4e7a6e72881770a64676c27c3c) Signed-off-by: Victor Vieux --- api/server/router/container/backend.go | 3 +- .../router/container/container_routes.go | 10 ++-- api/server/router/image/backend.go | 2 +- api/server/router/image/image_routes.go | 10 ++-- api/server/router/network/backend.go | 3 +- api/server/router/network/network_routes.go | 11 +---- api/server/router/volume/backend.go | 3 +- api/server/router/volume/volume_routes.go | 13 +---- api/swagger.yaml | 49 +++++++++++++------ api/types/types.go | 21 -------- cli/command/container/prune.go | 4 +- cli/command/image/prune.go | 9 ++-- cli/command/network/prune.go | 4 +- cli/command/volume/prune.go | 4 +- client/container_prune.go | 10 +++- client/image_prune.go | 10 +++- client/interface.go | 8 +-- client/network_prune.go | 14 +++++- client/utils.go | 20 +++++++- client/volume_prune.go | 10 +++- daemon/prune.go | 31 ++++++++---- integration-cli/docker_cli_prune_unix_test.go | 30 ++++++++++++ 22 files changed, 171 insertions(+), 108 deletions(-) diff --git a/api/server/router/container/backend.go b/api/server/router/container/backend.go index 5b4134eba5705694f75c71f25d1a7bd577ec045a..0d20188ccf4c5f63c2ff7badfb5123eb21a31f15 100644 --- a/api/server/router/container/backend.go +++ b/api/server/router/container/backend.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/pkg/archive" ) @@ -64,7 +65,7 @@ type attachBackend interface { // systemBackend includes functions to implement to provide system wide containers functionality type systemBackend interface { - ContainersPrune(config *types.ContainersPruneConfig) (*types.ContainersPruneReport, error) + ContainersPrune(pruneFilters filters.Args) (*types.ContainersPruneReport, error) } // Backend is all the methods that need to be implemented to provide container specific functionality. diff --git a/api/server/router/container/container_routes.go b/api/server/router/container/container_routes.go index 18bc50bcacf3eebe4c1c36e825437e1f6308d112..9c9bc0f8c3981f3a5246e78c7d3c524471d730a6 100644 --- a/api/server/router/container/container_routes.go +++ b/api/server/router/container/container_routes.go @@ -541,16 +541,12 @@ func (s *containerRouter) postContainersPrune(ctx context.Context, w http.Respon return err } - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - var cfg types.ContainersPruneConfig - if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { + pruneFilters, err := filters.FromParam(r.Form.Get("filters")) + if err != nil { return err } - pruneReport, err := s.backend.ContainersPrune(&cfg) + pruneReport, err := s.backend.ContainersPrune(pruneFilters) if err != nil { return err } diff --git a/api/server/router/image/backend.go b/api/server/router/image/backend.go index 5209d7e8798c7cd1a96f92331273c6a9e9409b15..19a67a5ed01af7b487bd8fa10058dc4ea7ac9b7f 100644 --- a/api/server/router/image/backend.go +++ b/api/server/router/image/backend.go @@ -29,7 +29,7 @@ type imageBackend interface { Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) LookupImage(name string) (*types.ImageInspect, error) TagImage(imageName, repository, tag string) error - ImagesPrune(config *types.ImagesPruneConfig) (*types.ImagesPruneReport, error) + ImagesPrune(pruneFilters filters.Args) (*types.ImagesPruneReport, error) } type importExportBackend interface { diff --git a/api/server/router/image/image_routes.go b/api/server/router/image/image_routes.go index 2b12503ddc52bca1328802329b036da01540bbcb..69403652a073c03bc51aa33749c1f38e6e6fa1b9 100644 --- a/api/server/router/image/image_routes.go +++ b/api/server/router/image/image_routes.go @@ -331,16 +331,12 @@ func (s *imageRouter) postImagesPrune(ctx context.Context, w http.ResponseWriter return err } - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - var cfg types.ImagesPruneConfig - if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { + pruneFilters, err := filters.FromParam(r.Form.Get("filters")) + if err != nil { return err } - pruneReport, err := s.backend.ImagesPrune(&cfg) + pruneReport, err := s.backend.ImagesPrune(pruneFilters) if err != nil { return err } diff --git a/api/server/router/network/backend.go b/api/server/router/network/backend.go index cf82398c93b9d88f2f302705fa2eafde6424e904..0d1dfb012327ce171b6d238f16f68dec6322d5c7 100644 --- a/api/server/router/network/backend.go +++ b/api/server/router/network/backend.go @@ -2,6 +2,7 @@ package network import ( "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" "github.com/docker/libnetwork" ) @@ -17,5 +18,5 @@ type Backend interface { ConnectContainerToNetwork(containerName, networkName string, endpointConfig *network.EndpointSettings) error DisconnectContainerFromNetwork(containerName string, networkName string, force bool) error DeleteNetwork(name string) error - NetworksPrune(config *types.NetworksPruneConfig) (*types.NetworksPruneReport, error) + NetworksPrune(pruneFilters filters.Args) (*types.NetworksPruneReport, error) } diff --git a/api/server/router/network/network_routes.go b/api/server/router/network/network_routes.go index 39b45e58bc573bf6fe014e6b5ecd1ffbea98264d..5b7b9738ebf1d44c7830eeeaa55cba68f61114b4 100644 --- a/api/server/router/network/network_routes.go +++ b/api/server/router/network/network_routes.go @@ -297,16 +297,7 @@ func (n *networkRouter) postNetworksPrune(ctx context.Context, w http.ResponseWr return err } - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - var cfg types.NetworksPruneConfig - if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { - return err - } - - pruneReport, err := n.backend.NetworksPrune(&cfg) + pruneReport, err := n.backend.NetworksPrune(filters.Args{}) if err != nil { return err } diff --git a/api/server/router/volume/backend.go b/api/server/router/volume/backend.go index 3a7608e0bda325aadd97da67026b2f9087130c35..180c06e5d324853963f1368b524c3700e57c6127 100644 --- a/api/server/router/volume/backend.go +++ b/api/server/router/volume/backend.go @@ -3,6 +3,7 @@ package volume import ( // TODO return types need to be refactored into pkg "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" ) // Backend is the methods that need to be implemented to provide @@ -12,5 +13,5 @@ type Backend interface { VolumeInspect(name string) (*types.Volume, error) VolumeCreate(name, driverName string, opts, labels map[string]string) (*types.Volume, error) VolumeRm(name string, force bool) error - VolumesPrune(config *types.VolumesPruneConfig) (*types.VolumesPruneReport, error) + VolumesPrune(pruneFilters filters.Args) (*types.VolumesPruneReport, error) } diff --git a/api/server/router/volume/volume_routes.go b/api/server/router/volume/volume_routes.go index e0398817c3e60e6a7e6eb39b60b599edd811bdb6..cfd4618a4d440ae176c78b19d47ed81b5ce61975 100644 --- a/api/server/router/volume/volume_routes.go +++ b/api/server/router/volume/volume_routes.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/docker/docker/api/server/httputils" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" volumetypes "github.com/docker/docker/api/types/volume" "golang.org/x/net/context" ) @@ -72,16 +72,7 @@ func (v *volumeRouter) postVolumesPrune(ctx context.Context, w http.ResponseWrit return err } - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - var cfg types.VolumesPruneConfig - if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { - return err - } - - pruneReport, err := v.backend.VolumesPrune(&cfg) + pruneReport, err := v.backend.VolumesPrune(filters.Args{}) if err != nil { return err } diff --git a/api/swagger.yaml b/api/swagger.yaml index f3f0f4b66e7bdb62f52f4a86a5741dcb5adbb4cb..8449707df8ab3f5d56ec77161134346efd08f184 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -4186,11 +4186,17 @@ paths: /containers/prune: post: summary: "Delete stopped containers" - consumes: - - "application/json" produces: - "application/json" operationId: "ContainerPrune" + parameters: + - name: "filters" + in: "query" + description: | + Filters to process on the prune list, encoded as JSON (a `map[string][]string`). + + Available filters: + type: "string" responses: 200: description: "No error" @@ -4848,21 +4854,20 @@ paths: /images/prune: post: summary: "Delete unused images" - consumes: - - "application/json" produces: - "application/json" operationId: "ImagePrune" parameters: - - name: "body" - in: "body" - schema: - type: "object" - properties: - DanglingOnly: - description: "Only delete unused *and* untagged images" - type: "boolean" - default: false + - name: "filters" + in: "query" + description: | + Filters to process on the prune list, encoded as JSON (a `map[string][]string`). + + Available filters: + - `dangling=` When set to `true` (or `1`), prune only + unused *and* untagged images. When set to `false` + (or `0`), all unused images are pruned. + type: "string" responses: 200: description: "No error" @@ -5944,11 +5949,17 @@ paths: /volumes/prune: post: summary: "Delete unused volumes" - consumes: - - "application/json" produces: - "application/json" operationId: "VolumePrune" + parameters: + - name: "filters" + in: "query" + description: | + Filters to process on the prune list, encoded as JSON (a `map[string][]string`). + + Available filters: + type: "string" responses: 200: description: "No error" @@ -6286,6 +6297,14 @@ paths: produces: - "application/json" operationId: "NetworkPrune" + parameters: + - name: "filters" + in: "query" + description: | + Filters to process on the prune list, encoded as JSON (a `map[string][]string`). + + Available filters: + type: "string" responses: 200: description: "No error" diff --git a/api/types/types.go b/api/types/types.go index 4a96ec556a1ff7742261a4d74fb3ce4bb1725b9a..a82c3e88ef50e07b4c050572f522a5bb62adf4a1 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -509,27 +509,6 @@ type DiskUsage struct { Volumes []*Volume } -// ImagesPruneConfig contains the configuration for Engine API: -// POST "/images/prune" -type ImagesPruneConfig struct { - DanglingOnly bool -} - -// ContainersPruneConfig contains the configuration for Engine API: -// POST "/images/prune" -type ContainersPruneConfig struct { -} - -// VolumesPruneConfig contains the configuration for Engine API: -// POST "/images/prune" -type VolumesPruneConfig struct { -} - -// NetworksPruneConfig contains the configuration for Engine API: -// POST "/networks/prune" -type NetworksPruneConfig struct { -} - // ContainersPruneReport contains the response for Engine API: // POST "/containers/prune" type ContainersPruneReport struct { diff --git a/cli/command/container/prune.go b/cli/command/container/prune.go index ec6b0e3147e538de2d79f4a0acb3779b7a86f38d..064f4c08e035218958f6dac760b87ff3c6022b3a 100644 --- a/cli/command/container/prune.go +++ b/cli/command/container/prune.go @@ -5,7 +5,7 @@ import ( "golang.org/x/net/context" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/cli" "github.com/docker/docker/cli/command" units "github.com/docker/go-units" @@ -52,7 +52,7 @@ func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed u return } - report, err := dockerCli.Client().ContainersPrune(context.Background(), types.ContainersPruneConfig{}) + report, err := dockerCli.Client().ContainersPrune(context.Background(), filters.Args{}) if err != nil { return } diff --git a/cli/command/image/prune.go b/cli/command/image/prune.go index ea84cda877b9138f42a017120779fe568ba21753..82c28fcf491e2b5fe8ed447a9a25324ae9ff880e 100644 --- a/cli/command/image/prune.go +++ b/cli/command/image/prune.go @@ -5,7 +5,7 @@ import ( "golang.org/x/net/context" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/cli" "github.com/docker/docker/cli/command" units "github.com/docker/go-units" @@ -54,6 +54,9 @@ Are you sure you want to continue?` ) func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) { + pruneFilters := filters.NewArgs() + pruneFilters.Add("dangling", fmt.Sprintf("%v", !opts.all)) + warning := danglingWarning if opts.all { warning = allImageWarning @@ -62,9 +65,7 @@ func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed u return } - report, err := dockerCli.Client().ImagesPrune(context.Background(), types.ImagesPruneConfig{ - DanglingOnly: !opts.all, - }) + report, err := dockerCli.Client().ImagesPrune(context.Background(), pruneFilters) if err != nil { return } diff --git a/cli/command/network/prune.go b/cli/command/network/prune.go index f2f8cc20c4ea284c63894422b284404ad9ddabc8..9f1979e6b5955e80f335e0c4f6f3740c08831a22 100644 --- a/cli/command/network/prune.go +++ b/cli/command/network/prune.go @@ -5,7 +5,7 @@ import ( "golang.org/x/net/context" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/cli" "github.com/docker/docker/cli/command" "github.com/spf13/cobra" @@ -50,7 +50,7 @@ func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (output string, e return } - report, err := dockerCli.Client().NetworksPrune(context.Background(), types.NetworksPruneConfig{}) + report, err := dockerCli.Client().NetworksPrune(context.Background(), filters.Args{}) if err != nil { return } diff --git a/cli/command/volume/prune.go b/cli/command/volume/prune.go index ac9c94451a6bfc30eda29497f944bafc4049d483..405fbeb29528fe2d0cc12364e9280eccc064940f 100644 --- a/cli/command/volume/prune.go +++ b/cli/command/volume/prune.go @@ -5,7 +5,7 @@ import ( "golang.org/x/net/context" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/cli" "github.com/docker/docker/cli/command" units "github.com/docker/go-units" @@ -52,7 +52,7 @@ func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed u return } - report, err := dockerCli.Client().VolumesPrune(context.Background(), types.VolumesPruneConfig{}) + report, err := dockerCli.Client().VolumesPrune(context.Background(), filters.Args{}) if err != nil { return } diff --git a/client/container_prune.go b/client/container_prune.go index 3eabe71a7fb0f14993fd9d6824bfeb5cd2aa524c..b5821708676b594c83583a0fb0ea925b4c1e7e33 100644 --- a/client/container_prune.go +++ b/client/container_prune.go @@ -5,18 +5,24 @@ import ( "fmt" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "golang.org/x/net/context" ) // ContainersPrune requests the daemon to delete unused data -func (cli *Client) ContainersPrune(ctx context.Context, cfg types.ContainersPruneConfig) (types.ContainersPruneReport, error) { +func (cli *Client) ContainersPrune(ctx context.Context, pruneFilters filters.Args) (types.ContainersPruneReport, error) { var report types.ContainersPruneReport if err := cli.NewVersionError("1.25", "container prune"); err != nil { return report, err } - serverResp, err := cli.post(ctx, "/containers/prune", nil, cfg, nil) + query, err := getFiltersQuery(pruneFilters) + if err != nil { + return report, err + } + + serverResp, err := cli.post(ctx, "/containers/prune", query, nil, nil) if err != nil { return report, err } diff --git a/client/image_prune.go b/client/image_prune.go index d5e69d5b19868354cef257eb049084691cc24c34..5ef98b7f0207ec0b0a156cec0a025c67115abe50 100644 --- a/client/image_prune.go +++ b/client/image_prune.go @@ -5,18 +5,24 @@ import ( "fmt" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "golang.org/x/net/context" ) // ImagesPrune requests the daemon to delete unused data -func (cli *Client) ImagesPrune(ctx context.Context, cfg types.ImagesPruneConfig) (types.ImagesPruneReport, error) { +func (cli *Client) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (types.ImagesPruneReport, error) { var report types.ImagesPruneReport if err := cli.NewVersionError("1.25", "image prune"); err != nil { return report, err } - serverResp, err := cli.post(ctx, "/images/prune", nil, cfg, nil) + query, err := getFiltersQuery(pruneFilters) + if err != nil { + return report, err + } + + serverResp, err := cli.post(ctx, "/images/prune", query, nil, nil) if err != nil { return report, err } diff --git a/client/interface.go b/client/interface.go index 0d722d90754015ccc725768422a2ff00f3401c7a..6319f34f1e74dba0f2a83caeb4b57d513b89a760 100644 --- a/client/interface.go +++ b/client/interface.go @@ -64,7 +64,7 @@ type ContainerAPIClient interface { ContainerWait(ctx context.Context, container string) (int64, error) CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, types.ContainerPathStat, error) CopyToContainer(ctx context.Context, container, path string, content io.Reader, options types.CopyToContainerOptions) error - ContainersPrune(ctx context.Context, cfg types.ContainersPruneConfig) (types.ContainersPruneReport, error) + ContainersPrune(ctx context.Context, pruneFilters filters.Args) (types.ContainersPruneReport, error) } // ImageAPIClient defines API client methods for the images @@ -82,7 +82,7 @@ type ImageAPIClient interface { ImageSearch(ctx context.Context, term string, options types.ImageSearchOptions) ([]registry.SearchResult, error) ImageSave(ctx context.Context, images []string) (io.ReadCloser, error) ImageTag(ctx context.Context, image, ref string) error - ImagesPrune(ctx context.Context, cfg types.ImagesPruneConfig) (types.ImagesPruneReport, error) + ImagesPrune(ctx context.Context, pruneFilter filters.Args) (types.ImagesPruneReport, error) } // NetworkAPIClient defines API client methods for the networks @@ -94,7 +94,7 @@ type NetworkAPIClient interface { NetworkInspectWithRaw(ctx context.Context, networkID string) (types.NetworkResource, []byte, error) NetworkList(ctx context.Context, options types.NetworkListOptions) ([]types.NetworkResource, error) NetworkRemove(ctx context.Context, networkID string) error - NetworksPrune(ctx context.Context, cfg types.NetworksPruneConfig) (types.NetworksPruneReport, error) + NetworksPrune(ctx context.Context, pruneFilter filters.Args) (types.NetworksPruneReport, error) } // NodeAPIClient defines API client methods for the nodes @@ -157,7 +157,7 @@ type VolumeAPIClient interface { VolumeInspectWithRaw(ctx context.Context, volumeID string) (types.Volume, []byte, error) VolumeList(ctx context.Context, filter filters.Args) (volumetypes.VolumesListOKBody, error) VolumeRemove(ctx context.Context, volumeID string, force bool) error - VolumesPrune(ctx context.Context, cfg types.VolumesPruneConfig) (types.VolumesPruneReport, error) + VolumesPrune(ctx context.Context, pruneFilter filters.Args) (types.VolumesPruneReport, error) } // SecretAPIClient defines API client methods for secrets diff --git a/client/network_prune.go b/client/network_prune.go index 01185f2e02b8aa5edf99696ba7338e63d93f439a..7352a7f0c5fb32bfc5afd8f5123c767ccf96d0c5 100644 --- a/client/network_prune.go +++ b/client/network_prune.go @@ -5,14 +5,24 @@ import ( "fmt" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "golang.org/x/net/context" ) // NetworksPrune requests the daemon to delete unused networks -func (cli *Client) NetworksPrune(ctx context.Context, cfg types.NetworksPruneConfig) (types.NetworksPruneReport, error) { +func (cli *Client) NetworksPrune(ctx context.Context, pruneFilters filters.Args) (types.NetworksPruneReport, error) { var report types.NetworksPruneReport - serverResp, err := cli.post(ctx, "/networks/prune", nil, cfg, nil) + if err := cli.NewVersionError("1.25", "network prune"); err != nil { + return report, err + } + + query, err := getFiltersQuery(pruneFilters) + if err != nil { + return report, err + } + + serverResp, err := cli.post(ctx, "/networks/prune", query, nil, nil) if err != nil { return report, err } diff --git a/client/utils.go b/client/utils.go index 03bf4c82fa42c8c1d37114193bde1815277840d5..23d520ecb8c7a6ce5ffd60ad6876b546e3088b8f 100644 --- a/client/utils.go +++ b/client/utils.go @@ -1,6 +1,10 @@ package client -import "regexp" +import ( + "github.com/docker/docker/api/types/filters" + "net/url" + "regexp" +) var headerRegexp = regexp.MustCompile(`\ADocker/.+\s\((.+)\)\z`) @@ -13,3 +17,17 @@ func getDockerOS(serverHeader string) string { } return osType } + +// getFiltersQuery returns a url query with "filters" query term, based on the +// filters provided. +func getFiltersQuery(f filters.Args) (url.Values, error) { + query := url.Values{} + if f.Len() > 0 { + filterJSON, err := filters.ToParam(f) + if err != nil { + return query, err + } + query.Set("filters", filterJSON) + } + return query, nil +} diff --git a/client/volume_prune.go b/client/volume_prune.go index ea4e234a30d3e3ff23377f8485bdd3fecf7736bf..a07e4ce637adfe000b579e041edca8ea5920991c 100644 --- a/client/volume_prune.go +++ b/client/volume_prune.go @@ -5,18 +5,24 @@ import ( "fmt" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "golang.org/x/net/context" ) // VolumesPrune requests the daemon to delete unused data -func (cli *Client) VolumesPrune(ctx context.Context, cfg types.VolumesPruneConfig) (types.VolumesPruneReport, error) { +func (cli *Client) VolumesPrune(ctx context.Context, pruneFilters filters.Args) (types.VolumesPruneReport, error) { var report types.VolumesPruneReport if err := cli.NewVersionError("1.25", "volume prune"); err != nil { return report, err } - serverResp, err := cli.post(ctx, "/volumes/prune", nil, cfg, nil) + query, err := getFiltersQuery(pruneFilters) + if err != nil { + return report, err + } + + serverResp, err := cli.post(ctx, "/volumes/prune", query, nil, nil) if err != nil { return report, err } diff --git a/daemon/prune.go b/daemon/prune.go index 953a568d9153d597fc52a3c3d404f4e7fa43c451..a693beb4e1078419ccbddd54a8381496f914bc7f 100644 --- a/daemon/prune.go +++ b/daemon/prune.go @@ -1,11 +1,13 @@ package daemon import ( + "fmt" "regexp" "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" @@ -16,7 +18,7 @@ import ( ) // ContainersPrune removes unused containers -func (daemon *Daemon) ContainersPrune(config *types.ContainersPruneConfig) (*types.ContainersPruneReport, error) { +func (daemon *Daemon) ContainersPrune(pruneFilters filters.Args) (*types.ContainersPruneReport, error) { rep := &types.ContainersPruneReport{} allContainers := daemon.List() @@ -40,7 +42,7 @@ func (daemon *Daemon) ContainersPrune(config *types.ContainersPruneConfig) (*typ } // VolumesPrune removes unused local volumes -func (daemon *Daemon) VolumesPrune(config *types.VolumesPruneConfig) (*types.VolumesPruneReport, error) { +func (daemon *Daemon) VolumesPrune(pruneFilters filters.Args) (*types.VolumesPruneReport, error) { rep := &types.VolumesPruneReport{} pruneVols := func(v volume.Volume) error { @@ -70,11 +72,20 @@ func (daemon *Daemon) VolumesPrune(config *types.VolumesPruneConfig) (*types.Vol } // ImagesPrune removes unused images -func (daemon *Daemon) ImagesPrune(config *types.ImagesPruneConfig) (*types.ImagesPruneReport, error) { +func (daemon *Daemon) ImagesPrune(pruneFilters filters.Args) (*types.ImagesPruneReport, error) { rep := &types.ImagesPruneReport{} + danglingOnly := true + if pruneFilters.Include("dangling") { + if pruneFilters.ExactMatch("dangling", "false") || pruneFilters.ExactMatch("dangling", "0") { + danglingOnly = false + } else if !pruneFilters.ExactMatch("dangling", "true") && !pruneFilters.ExactMatch("dangling", "1") { + return nil, fmt.Errorf("Invalid filter 'dangling=%s'", pruneFilters.Get("dangling")) + } + } + var allImages map[image.ID]*image.Image - if config.DanglingOnly { + if danglingOnly { allImages = daemon.imageStore.Heads() } else { allImages = daemon.imageStore.Map() @@ -106,7 +117,7 @@ func (daemon *Daemon) ImagesPrune(config *types.ImagesPruneConfig) (*types.Image deletedImages := []types.ImageDelete{} refs := daemon.referenceStore.References(dgst) if len(refs) > 0 { - if config.DanglingOnly { + if danglingOnly { // Not a dangling image continue } @@ -156,7 +167,7 @@ func (daemon *Daemon) ImagesPrune(config *types.ImagesPruneConfig) (*types.Image } // localNetworksPrune removes unused local networks -func (daemon *Daemon) localNetworksPrune(config *types.NetworksPruneConfig) (*types.NetworksPruneReport, error) { +func (daemon *Daemon) localNetworksPrune(pruneFilters filters.Args) (*types.NetworksPruneReport, error) { rep := &types.NetworksPruneReport{} var err error // When the function returns true, the walk will stop. @@ -177,7 +188,7 @@ func (daemon *Daemon) localNetworksPrune(config *types.NetworksPruneConfig) (*ty } // clusterNetworksPrune removes unused cluster networks -func (daemon *Daemon) clusterNetworksPrune(config *types.NetworksPruneConfig) (*types.NetworksPruneReport, error) { +func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.NetworksPruneReport, error) { rep := &types.NetworksPruneReport{} cluster := daemon.GetCluster() networks, err := cluster.GetNetworks() @@ -207,15 +218,15 @@ func (daemon *Daemon) clusterNetworksPrune(config *types.NetworksPruneConfig) (* } // NetworksPrune removes unused networks -func (daemon *Daemon) NetworksPrune(config *types.NetworksPruneConfig) (*types.NetworksPruneReport, error) { +func (daemon *Daemon) NetworksPrune(pruneFilters filters.Args) (*types.NetworksPruneReport, error) { rep := &types.NetworksPruneReport{} - clusterRep, err := daemon.clusterNetworksPrune(config) + clusterRep, err := daemon.clusterNetworksPrune(pruneFilters) if err != nil { logrus.Warnf("could not remove cluster networks: %v", err) } else { rep.NetworksDeleted = append(rep.NetworksDeleted, clusterRep.NetworksDeleted...) } - localRep, err := daemon.localNetworksPrune(config) + localRep, err := daemon.localNetworksPrune(pruneFilters) if err != nil { logrus.Warnf("could not remove local networks: %v", err) } else { diff --git a/integration-cli/docker_cli_prune_unix_test.go b/integration-cli/docker_cli_prune_unix_test.go index 5585cab302498fb6db152d7085681a02a000bf1f..dabbc7208100f195946b08b2d45056ed434e1c11 100644 --- a/integration-cli/docker_cli_prune_unix_test.go +++ b/integration-cli/docker_cli_prune_unix_test.go @@ -59,3 +59,33 @@ func (s *DockerSwarmSuite) TestPruneNetwork(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0) pruneNetworkAndVerify(c, d, []string{}, []string{"n1", "n3"}) } + +func (s *DockerDaemonSuite) TestPruneImageDangling(c *check.C) { + c.Assert(s.d.StartWithBusybox(), checker.IsNil) + + out, _, err := s.d.buildImageWithOut("test", + `FROM busybox + LABEL foo=bar`, true, "-q") + c.Assert(err, checker.IsNil) + id := strings.TrimSpace(out) + + out, err = s.d.Cmd("images", "-q", "--no-trunc") + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Contains, id) + + out, err = s.d.Cmd("image", "prune", "--force") + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id) + + out, err = s.d.Cmd("images", "-q", "--no-trunc") + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Contains, id) + + out, err = s.d.Cmd("image", "prune", "--force", "--all") + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Contains, id) + + out, err = s.d.Cmd("images", "-q", "--no-trunc") + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id) +} From 696130c949df0d49adc22c737acbda421b669d28 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 28 Nov 2016 12:18:15 -0800 Subject: [PATCH 02/68] Fix issue where secret ID is masked by name This fix tries to address the issue in 28884 where it is possible to mask the secret ID by name. The reason was that searching a secret is based on name. However, searching a secret should be done based on: - Full ID - Full Name - Partial ID (prefix) This fix addresses the issue by changing related implementation in `getCliRequestedSecretIDs()` An integration test has been added to cover the changes. This fix fixes 28884 Signed-off-by: Yong Tang (cherry picked from commit 3638ca4d14bcca9bc924f21314e4a1020cd5172f) Signed-off-by: Victor Vieux --- cli/command/secret/utils.go | 64 +++++++++++++------ .../docker_cli_secret_create_test.go | 58 +++++++++++++++++ 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/cli/command/secret/utils.go b/cli/command/secret/utils.go index 0134853e09167fa821030469d1e86e2e6ece852f..42493896ca74e197f758500edde16a33c9a23880 100644 --- a/cli/command/secret/utils.go +++ b/cli/command/secret/utils.go @@ -1,6 +1,9 @@ package secret import ( + "fmt" + "strings" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" @@ -8,10 +11,11 @@ import ( "golang.org/x/net/context" ) -func getSecretsByName(ctx context.Context, client client.APIClient, names []string) ([]swarm.Secret, error) { +func getSecretsByNameOrIDPrefixes(ctx context.Context, client client.APIClient, terms []string) ([]swarm.Secret, error) { args := filters.NewArgs() - for _, n := range names { + for _, n := range terms { args.Add("names", n) + args.Add("id", n) } return client.SecretList(ctx, types.SecretListOptions{ @@ -19,29 +23,53 @@ func getSecretsByName(ctx context.Context, client client.APIClient, names []stri }) } -func getCliRequestedSecretIDs(ctx context.Context, client client.APIClient, names []string) ([]string, error) { - ids := names - - // attempt to lookup secret by name - secrets, err := getSecretsByName(ctx, client, ids) +func getCliRequestedSecretIDs(ctx context.Context, client client.APIClient, terms []string) ([]string, error) { + secrets, err := getSecretsByNameOrIDPrefixes(ctx, client, terms) if err != nil { return nil, err } - lookup := make(map[string]struct{}) - for _, id := range ids { - lookup[id] = struct{}{} - } - if len(secrets) > 0 { - ids = []string{} - - for _, s := range secrets { - if _, ok := lookup[s.Spec.Annotations.Name]; ok { - ids = append(ids, s.ID) + found := make(map[string]struct{}) + next: + for _, term := range terms { + // attempt to lookup secret by full ID + for _, s := range secrets { + if s.ID == term { + found[s.ID] = struct{}{} + continue next + } + } + // attempt to lookup secret by full name + for _, s := range secrets { + if s.Spec.Annotations.Name == term { + found[s.ID] = struct{}{} + continue next + } + } + // attempt to lookup secret by partial ID (prefix) + // return error if more than one matches found (ambiguous) + n := 0 + for _, s := range secrets { + if strings.HasPrefix(s.ID, term) { + found[s.ID] = struct{}{} + n++ + } + } + if n > 1 { + return nil, fmt.Errorf("secret %s is ambiguous (%d matches found)", term, n) } } + + // We already collected all the IDs found. + // Now we will remove duplicates by converting the map to slice + ids := []string{} + for id := range found { + ids = append(ids, id) + } + + return ids, nil } - return ids, nil + return terms, nil } diff --git a/integration-cli/docker_cli_secret_create_test.go b/integration-cli/docker_cli_secret_create_test.go index 3126a0db144874b05f70abc66978232596c9655d..9c45f8a0ae0d7c9da5bcf7c25dfef1b27661a9f5 100644 --- a/integration-cli/docker_cli_secret_create_test.go +++ b/integration-cli/docker_cli_secret_create_test.go @@ -46,3 +46,61 @@ func (s *DockerSwarmSuite) TestSecretCreateWithLabels(c *check.C) { c.Assert(secret.Spec.Labels["key1"], checker.Equals, "value1") c.Assert(secret.Spec.Labels["key2"], checker.Equals, "value2") } + +// Test case for 28884 +func (s *DockerSwarmSuite) TestSecretCreateResolve(c *check.C) { + d := s.AddDaemon(c, true, true) + + name := "foo" + id := d.createSecret(c, swarm.SecretSpec{ + swarm.Annotations{ + Name: name, + }, + []byte("foo"), + }) + c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) + + fake := d.createSecret(c, swarm.SecretSpec{ + swarm.Annotations{ + Name: id, + }, + []byte("fake foo"), + }) + c.Assert(fake, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", fake)) + + out, err := d.Cmd("secret", "ls") + c.Assert(err, checker.IsNil) + c.Assert(out, checker.Contains, name) + c.Assert(out, checker.Contains, fake) + + out, err = d.Cmd("secret", "rm", id) + c.Assert(out, checker.Contains, id) + + // Fake one will remain + out, err = d.Cmd("secret", "ls") + c.Assert(err, checker.IsNil) + c.Assert(out, checker.Not(checker.Contains), name) + c.Assert(out, checker.Contains, fake) + + // Remove based on name prefix of the fake one + // (which is the same as the ID of foo one) should not work + // as search is only done based on: + // - Full ID + // - Full Name + // - Partial ID (prefix) + out, err = d.Cmd("secret", "rm", id[:5]) + c.Assert(out, checker.Not(checker.Contains), id) + out, err = d.Cmd("secret", "ls") + c.Assert(err, checker.IsNil) + c.Assert(out, checker.Not(checker.Contains), name) + c.Assert(out, checker.Contains, fake) + + // Remove based on ID prefix of the fake one should succeed + out, err = d.Cmd("secret", "rm", fake[:5]) + c.Assert(out, checker.Contains, fake) + out, err = d.Cmd("secret", "ls") + c.Assert(err, checker.IsNil) + c.Assert(out, checker.Not(checker.Contains), name) + c.Assert(out, checker.Not(checker.Contains), id) + c.Assert(out, checker.Not(checker.Contains), fake) +} From 728296b9eab1298565d27e8efb02c85d83e18a6d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 23 Nov 2016 17:29:21 -0800 Subject: [PATCH 03/68] refactor plugin install Signed-off-by: Victor Vieux (cherry picked from commit fa3b61a28f55d84afbbb978785ce9632123d12fa) Signed-off-by: Victor Vieux --- api/server/router/plugin/backend.go | 3 +- api/server/router/plugin/plugin.go | 3 +- api/server/router/plugin/plugin_routes.go | 57 ++++++---- client/plugin_inspect.go | 2 +- client/plugin_install.go | 33 ++++-- plugin/backend_linux.go | 132 ++++++++++++++++++---- plugin/backend_unsupported.go | 9 +- plugin/distribution/pull.go | 1 - plugin/v2/plugin.go | 47 -------- 9 files changed, 176 insertions(+), 111 deletions(-) diff --git a/api/server/router/plugin/backend.go b/api/server/router/plugin/backend.go index cc931fe411a102b0385e511696206ff752b3c6bd..fba42f3e81a344a7e7a8f9de5f7e11ceb995c186 100644 --- a/api/server/router/plugin/backend.go +++ b/api/server/router/plugin/backend.go @@ -16,7 +16,8 @@ type Backend interface { Inspect(name string) (enginetypes.Plugin, error) Remove(name string, config *enginetypes.PluginRmConfig) error Set(name string, args []string) error - Pull(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) (enginetypes.PluginPrivileges, error) + Privileges(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) (enginetypes.PluginPrivileges, error) + Pull(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges) error Push(name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) error CreateFromContext(ctx context.Context, tarCtx io.Reader, options *enginetypes.PluginCreateOptions) error } diff --git a/api/server/router/plugin/plugin.go b/api/server/router/plugin/plugin.go index 15bdcb3cd5525624e1272d85a523b8105d689f88..3f6ff566c84dcab91991e4473cf080f4174602af 100644 --- a/api/server/router/plugin/plugin.go +++ b/api/server/router/plugin/plugin.go @@ -25,7 +25,8 @@ func (r *pluginRouter) Routes() []router.Route { func (r *pluginRouter) initRoutes() { r.routes = []router.Route{ router.NewGetRoute("/plugins", r.listPlugins), - router.NewGetRoute("/plugins/{name:.*}", r.inspectPlugin), + router.NewGetRoute("/plugins/{name:.*}/json", r.inspectPlugin), + router.NewGetRoute("/plugins/privileges", r.getPrivileges), router.NewDeleteRoute("/plugins/{name:.*}", r.removePlugin), router.NewPostRoute("/plugins/{name:.*}/enable", r.enablePlugin), // PATCH? router.NewPostRoute("/plugins/{name:.*}/disable", r.disablePlugin), diff --git a/api/server/router/plugin/plugin_routes.go b/api/server/router/plugin/plugin_routes.go index ffa05dc984cac496e4bf2377265fb5bbd41cb34e..6a2ba7dc4fb28b2c0055e3f72e8e957796a726e1 100644 --- a/api/server/router/plugin/plugin_routes.go +++ b/api/server/router/plugin/plugin_routes.go @@ -12,20 +12,17 @@ import ( "golang.org/x/net/context" ) -func (pr *pluginRouter) pullPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := httputils.ParseForm(r); err != nil { - return err - } +func parseHeaders(headers http.Header) (map[string][]string, *types.AuthConfig) { metaHeaders := map[string][]string{} - for k, v := range r.Header { + for k, v := range headers { if strings.HasPrefix(k, "X-Meta-") { metaHeaders[k] = v } } // Get X-Registry-Auth - authEncoded := r.Header.Get("X-Registry-Auth") + authEncoded := headers.Get("X-Registry-Auth") authConfig := &types.AuthConfig{} if authEncoded != "" { authJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authEncoded)) @@ -34,13 +31,42 @@ func (pr *pluginRouter) pullPlugin(ctx context.Context, w http.ResponseWriter, r } } - privileges, err := pr.backend.Pull(r.FormValue("name"), metaHeaders, authConfig) + return metaHeaders, authConfig +} + +func (pr *pluginRouter) getPrivileges(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + metaHeaders, authConfig := parseHeaders(r.Header) + + privileges, err := pr.backend.Privileges(r.FormValue("name"), metaHeaders, authConfig) if err != nil { return err } return httputils.WriteJSON(w, http.StatusOK, privileges) } +func (pr *pluginRouter) pullPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + var privileges types.PluginPrivileges + if err := json.NewDecoder(r.Body).Decode(&privileges); err != nil { + return err + } + + metaHeaders, authConfig := parseHeaders(r.Header) + + if err := pr.backend.Pull(r.FormValue("name"), metaHeaders, authConfig, privileges); err != nil { + return err + } + w.WriteHeader(http.StatusCreated) + return nil +} + func (pr *pluginRouter) createPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := httputils.ParseForm(r); err != nil { return err @@ -52,6 +78,7 @@ func (pr *pluginRouter) createPlugin(ctx context.Context, w http.ResponseWriter, if err := pr.backend.CreateFromContext(ctx, r.Body, options); err != nil { return err } + //TODO: send progress bar w.WriteHeader(http.StatusNoContent) return nil } @@ -92,22 +119,8 @@ func (pr *pluginRouter) pushPlugin(ctx context.Context, w http.ResponseWriter, r return err } - metaHeaders := map[string][]string{} - for k, v := range r.Header { - if strings.HasPrefix(k, "X-Meta-") { - metaHeaders[k] = v - } - } + metaHeaders, authConfig := parseHeaders(r.Header) - // Get X-Registry-Auth - authEncoded := r.Header.Get("X-Registry-Auth") - authConfig := &types.AuthConfig{} - if authEncoded != "" { - authJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authEncoded)) - if err := json.NewDecoder(authJSON).Decode(authConfig); err != nil { - authConfig = &types.AuthConfig{} - } - } return pr.backend.Push(vars["name"], metaHeaders, authConfig) } diff --git a/client/plugin_inspect.go b/client/plugin_inspect.go index 72900a131006c271b768c509a412f0dfa05675bb..89f39ee2c6487e920788da9bd7e6296da4f669a9 100644 --- a/client/plugin_inspect.go +++ b/client/plugin_inspect.go @@ -12,7 +12,7 @@ import ( // PluginInspectWithRaw inspects an existing plugin func (cli *Client) PluginInspectWithRaw(ctx context.Context, name string) (*types.Plugin, []byte, error) { - resp, err := cli.get(ctx, "/plugins/"+name, nil, nil) + resp, err := cli.get(ctx, "/plugins/"+name+"/json", nil, nil) if err != nil { if resp.statusCode == http.StatusNotFound { return nil, nil, pluginNotFoundError{name} diff --git a/client/plugin_install.go b/client/plugin_install.go index f73362ccd35e0ed9c9a4ab929eb4e3c416a1675a..e7b67f20515ac2026563eca918138120e8eaa7e7 100644 --- a/client/plugin_install.go +++ b/client/plugin_install.go @@ -14,27 +14,21 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types // FIXME(vdemeester) name is a ref, we might want to parse/validate it here. query := url.Values{} query.Set("name", name) - resp, err := cli.tryPluginPull(ctx, query, options.RegistryAuth) + resp, err := cli.tryPluginPrivileges(ctx, query, options.RegistryAuth) if resp.statusCode == http.StatusUnauthorized && options.PrivilegeFunc != nil { newAuthHeader, privilegeErr := options.PrivilegeFunc() if privilegeErr != nil { ensureReaderClosed(resp) return privilegeErr } - resp, err = cli.tryPluginPull(ctx, query, newAuthHeader) + options.RegistryAuth = newAuthHeader + resp, err = cli.tryPluginPrivileges(ctx, query, options.RegistryAuth) } if err != nil { ensureReaderClosed(resp) return err } - defer func() { - if err != nil { - delResp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil) - ensureReaderClosed(delResp) - } - }() - var privileges types.PluginPrivileges if err := json.NewDecoder(resp.body).Decode(&privileges); err != nil { ensureReaderClosed(resp) @@ -52,6 +46,18 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types } } + _, err = cli.tryPluginPull(ctx, query, privileges, options.RegistryAuth) + if err != nil { + return err + } + + defer func() { + if err != nil { + delResp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil) + ensureReaderClosed(delResp) + } + }() + if len(options.Args) > 0 { if err := cli.PluginSet(ctx, name, options.Args); err != nil { return err @@ -65,7 +71,12 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types return cli.PluginEnable(ctx, name, types.PluginEnableOptions{Timeout: 0}) } -func (cli *Client) tryPluginPull(ctx context.Context, query url.Values, registryAuth string) (serverResponse, error) { +func (cli *Client) tryPluginPrivileges(ctx context.Context, query url.Values, registryAuth string) (serverResponse, error) { + headers := map[string][]string{"X-Registry-Auth": {registryAuth}} + return cli.get(ctx, "/plugins/privileges", query, headers) +} + +func (cli *Client) tryPluginPull(ctx context.Context, query url.Values, privileges types.PluginPrivileges, registryAuth string) (serverResponse, error) { headers := map[string][]string{"X-Registry-Auth": {registryAuth}} - return cli.post(ctx, "/plugins/pull", query, nil, headers) + return cli.post(ctx, "/plugins/pull", query, privileges, headers) } diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go index 5ab9b4d6afc6b48afdd2bc9f52e5011d645c2753..4b4fd11a65e4a6d56015aa2754e2c8de9a395c57 100644 --- a/plugin/backend_linux.go +++ b/plugin/backend_linux.go @@ -5,12 +5,14 @@ package plugin import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" "net/http" "os" "path/filepath" + "reflect" "regexp" "github.com/Sirupsen/logrus" @@ -87,59 +89,139 @@ func (pm *Manager) Inspect(refOrID string) (tp types.Plugin, err error) { return tp, fmt.Errorf("no such plugin name or ID associated with %q", refOrID) } -func (pm *Manager) pull(ref reference.Named, metaHeader http.Header, authConfig *types.AuthConfig, pluginID string) (types.PluginPrivileges, error) { +func (pm *Manager) pull(name string, metaHeader http.Header, authConfig *types.AuthConfig) (reference.Named, distribution.PullData, error) { + ref, err := distribution.GetRef(name) + if err != nil { + logrus.Debugf("error in distribution.GetRef: %v", err) + return nil, nil, err + } + name = ref.String() + + if p, _ := pm.pluginStore.GetByName(name); p != nil { + logrus.Debug("plugin already exists") + return nil, nil, fmt.Errorf("%s exists", name) + } + pd, err := distribution.Pull(ref, pm.registryService, metaHeader, authConfig) if err != nil { logrus.Debugf("error in distribution.Pull(): %v", err) - return nil, err + return nil, nil, err } + return ref, pd, nil +} - if err := distribution.WritePullData(pd, filepath.Join(pm.libRoot, pluginID), true); err != nil { - logrus.Debugf("error in distribution.WritePullData(): %v", err) +func computePrivileges(pd distribution.PullData) (types.PluginPrivileges, error) { + config, err := pd.Config() + if err != nil { return nil, err } - tag := distribution.GetTag(ref) - p := v2.NewPlugin(ref.Name(), pluginID, pm.runRoot, pm.libRoot, tag) - if err := p.InitPlugin(); err != nil { + var c types.PluginConfig + if err := json.Unmarshal(config, &c); err != nil { return nil, err } - pm.pluginStore.Add(p) - pm.pluginEventLogger(pluginID, ref.String(), "pull") - return p.ComputePrivileges(), nil + var privileges types.PluginPrivileges + if c.Network.Type != "null" && c.Network.Type != "bridge" { + privileges = append(privileges, types.PluginPrivilege{ + Name: "network", + Description: "permissions to access a network", + Value: []string{c.Network.Type}, + }) + } + for _, mount := range c.Mounts { + if mount.Source != nil { + privileges = append(privileges, types.PluginPrivilege{ + Name: "mount", + Description: "host path to mount", + Value: []string{*mount.Source}, + }) + } + } + for _, device := range c.Linux.Devices { + if device.Path != nil { + privileges = append(privileges, types.PluginPrivilege{ + Name: "device", + Description: "host device to access", + Value: []string{*device.Path}, + }) + } + } + if c.Linux.DeviceCreation { + privileges = append(privileges, types.PluginPrivilege{ + Name: "device-creation", + Description: "allow creating devices inside plugin", + Value: []string{"true"}, + }) + } + if len(c.Linux.Capabilities) > 0 { + privileges = append(privileges, types.PluginPrivilege{ + Name: "capabilities", + Description: "list of additional capabilities required", + Value: c.Linux.Capabilities, + }) + } + + return privileges, nil } -// Pull pulls a plugin and computes the privileges required to install it. -func (pm *Manager) Pull(name string, metaHeader http.Header, authConfig *types.AuthConfig) (types.PluginPrivileges, error) { - ref, err := distribution.GetRef(name) +// Privileges pulls a plugin config and computes the privileges required to install it. +func (pm *Manager) Privileges(name string, metaHeader http.Header, authConfig *types.AuthConfig) (types.PluginPrivileges, error) { + _, pd, err := pm.pull(name, metaHeader, authConfig) if err != nil { - logrus.Debugf("error in distribution.GetRef: %v", err) return nil, err } - name = ref.String() + return computePrivileges(pd) +} - if p, _ := pm.pluginStore.GetByName(name); p != nil { - logrus.Debug("plugin already exists") - return nil, fmt.Errorf("%s exists", name) +// Pull pulls a plugin, check if the correct privileges are provided and install the plugin. +func (pm *Manager) Pull(name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges) (err error) { + ref, pd, err := pm.pull(name, metaHeader, authConfig) + if err != nil { + return err + } + + requiredPrivileges, err := computePrivileges(pd) + if err != nil { + return err + } + + if !reflect.DeepEqual(privileges, requiredPrivileges) { + return errors.New("incorrect privileges") } pluginID := stringid.GenerateNonCryptoID() pluginDir := filepath.Join(pm.libRoot, pluginID) if err := os.MkdirAll(pluginDir, 0755); err != nil { logrus.Debugf("error in MkdirAll: %v", err) - return nil, err + return err } - priv, err := pm.pull(ref, metaHeader, authConfig, pluginID) - if err != nil { - if err := os.RemoveAll(pluginDir); err != nil { - logrus.Warnf("unable to remove %q from failed plugin pull: %v", pluginDir, err) + defer func() { + if err != nil { + if delErr := os.RemoveAll(pluginDir); delErr != nil { + logrus.Warnf("unable to remove %q from failed plugin pull: %v", pluginDir, delErr) + } } - return nil, err + }() + + err = distribution.WritePullData(pd, filepath.Join(pm.libRoot, pluginID), true) + if err != nil { + logrus.Debugf("error in distribution.WritePullData(): %v", err) + return err + } + + tag := distribution.GetTag(ref) + p := v2.NewPlugin(ref.Name(), pluginID, pm.runRoot, pm.libRoot, tag) + err = p.InitPlugin() + if err != nil { + return err } + pm.pluginStore.Add(p) - return priv, nil + pm.pluginEventLogger(pluginID, ref.String(), "pull") + + return nil } // List displays the list of plugins and associated metadata. diff --git a/plugin/backend_unsupported.go b/plugin/backend_unsupported.go index e54994fe7556c4e614fcd56d0210cdcdc6daf26f..0e07cd679ae8b7cc5bc0764e04590223c3c6eb13 100644 --- a/plugin/backend_unsupported.go +++ b/plugin/backend_unsupported.go @@ -28,11 +28,16 @@ func (pm *Manager) Inspect(name string) (tp types.Plugin, err error) { return tp, errNotSupported } -// Pull pulls a plugin and computes the privileges required to install it. -func (pm *Manager) Pull(name string, metaHeader http.Header, authConfig *types.AuthConfig) (types.PluginPrivileges, error) { +// Privileges pulls a plugin config and computes the privileges required to install it. +func (pm *Manager) Privileges(name string, metaHeaders http.Header, authConfig *types.AuthConfig) (types.PluginPrivileges, error) { return nil, errNotSupported } +// Pull pulls a plugin, check if the correct privileges are provided and install the plugin. +func (pm *Manager) Pull(name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges) error { + return errNotSupported +} + // List displays the list of plugins and associated metadata. func (pm *Manager) List() ([]types.Plugin, error) { return nil, errNotSupported diff --git a/plugin/distribution/pull.go b/plugin/distribution/pull.go index 5694be0573ef17efdfa5176bf39a630939973b20..dba750f2a5e672edf0627b58b96d3ec28b5bb4cc 100644 --- a/plugin/distribution/pull.go +++ b/plugin/distribution/pull.go @@ -178,7 +178,6 @@ func WritePullData(pd PullData, dest string, extract bool) error { return err } logrus.Debugf("%#v", p) - if err := os.MkdirAll(dest, 0700); err != nil { return err } diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go index 4679498ee6c48caf9a34ed99076d564b94a0ed29..7ea115cb3978d94f59e0ab53748b1fda2506fef2 100644 --- a/plugin/v2/plugin.go +++ b/plugin/v2/plugin.go @@ -216,53 +216,6 @@ next: return p.writeSettings() } -// ComputePrivileges takes the config file and computes the list of access necessary -// for the plugin on the host. -func (p *Plugin) ComputePrivileges() types.PluginPrivileges { - c := p.PluginObj.Config - var privileges types.PluginPrivileges - if c.Network.Type != "null" && c.Network.Type != "bridge" { - privileges = append(privileges, types.PluginPrivilege{ - Name: "network", - Description: "permissions to access a network", - Value: []string{c.Network.Type}, - }) - } - for _, mount := range c.Mounts { - if mount.Source != nil { - privileges = append(privileges, types.PluginPrivilege{ - Name: "mount", - Description: "host path to mount", - Value: []string{*mount.Source}, - }) - } - } - for _, device := range c.Linux.Devices { - if device.Path != nil { - privileges = append(privileges, types.PluginPrivilege{ - Name: "device", - Description: "host device to access", - Value: []string{*device.Path}, - }) - } - } - if c.Linux.DeviceCreation { - privileges = append(privileges, types.PluginPrivilege{ - Name: "device-creation", - Description: "allow creating devices inside plugin", - Value: []string{"true"}, - }) - } - if len(c.Linux.Capabilities) > 0 { - privileges = append(privileges, types.PluginPrivilege{ - Name: "capabilities", - Description: "list of additional capabilities required", - Value: c.Linux.Capabilities, - }) - } - return privileges -} - // IsEnabled returns the active state of the plugin. func (p *Plugin) IsEnabled() bool { p.RLock() From 8bed67c368d8698580c25848b589ac7e8439eb52 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 29 Nov 2016 16:31:33 -0800 Subject: [PATCH 04/68] update docs Signed-off-by: Victor Vieux (cherry picked from commit 30db51c169497f6d1f6d9517f25c237dd61182de) Signed-off-by: Victor Vieux --- api/swagger.yaml | 67 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 8449707df8ab3f5d56ec77161134346efd08f184..766d6965d473926c6b764c98948d70d6917a8bfd 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -6415,14 +6415,10 @@ paths: $ref: "#/definitions/ErrorResponse" tags: ["Plugin"] - /plugins/pull: - post: - summary: "Install a plugin" - operationId: "PluginPull" - description: | - Pulls and installs a plugin. After the plugin is installed, it can be enabled using the [`POST /plugins/{name}/enable` endpoint](#operation/PluginEnable). - produces: - - "application/json" + /plugins/privileges: + get: + summary: "Get plugin privileges" + operationId: "GetPluginPrivileges" responses: 200: description: "no error" @@ -6457,6 +6453,30 @@ paths: description: "server error" schema: $ref: "#/definitions/ErrorResponse" + parameters: + - name: "name" + in: "query" + description: "The name of the plugin. The `:latest` tag is optional, and is the default if omitted." + required: true + type: "string" + tags: + - "Plugin" + + /plugins/pull: + post: + summary: "Install a plugin" + operationId: "PluginPull" + description: | + Pulls and installs a plugin. After the plugin is installed, it can be enabled using the [`POST /plugins/{name}/enable` endpoint](#operation/PostPluginsEnable). + produces: + - "application/json" + responses: + 204: + description: "no error" + 500: + description: "server error" + schema: + $ref: "#/definitions/ErrorResponse" parameters: - name: "name" in: "query" @@ -6470,8 +6490,37 @@ paths: in: "header" description: "A base64-encoded auth configuration to use when pulling a plugin from a registry. [See the authentication section for details.](#section/Authentication)" type: "string" + - name: "body" + in: "body" + schema: + type: "array" + items: + description: "Describes a permission accepted by the user upon installing the plugin." + type: "object" + properties: + Name: + type: "string" + Description: + type: "string" + Value: + type: "array" + items: + type: "string" + example: + - Name: "network" + Description: "" + Value: + - "host" + - Name: "mount" + Description: "" + Value: + - "/data" + - Name: "device" + Description: "" + Value: + - "/dev/cpu_dma_latency" tags: ["Plugin"] - /plugins/{name}: + /plugins/{name}/json: get: summary: "Inspect a plugin" operationId: "PluginInspect" From e26de82e5415d9faae3b3916836b2f34a6952763 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 30 Nov 2016 05:48:44 +0000 Subject: [PATCH 05/68] improve TestServiceLogs for the goroutine issue #28915 Signed-off-by: Akihiro Suda (cherry picked from commit f8a93d0c9d157dddc4e4d4d9c43a6fe7c7c0c242) Signed-off-by: Victor Vieux --- ...cker_cli_service_logs_experimental_test.go | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/integration-cli/docker_cli_service_logs_experimental_test.go b/integration-cli/docker_cli_service_logs_experimental_test.go index e5003282e6ddde11cb8de940385c730aced5aec2..c2216543d7373667edc41228aa746aab9181f3cc 100644 --- a/integration-cli/docker_cli_service_logs_experimental_test.go +++ b/integration-cli/docker_cli_service_logs_experimental_test.go @@ -23,19 +23,29 @@ func (s *DockerSwarmSuite) TestServiceLogs(c *check.C) { d := s.AddDaemon(c, true, true) - name := "TestServiceLogs" + // we have multiple services here for detecting the goroutine issue #28915 + services := map[string]string{ + "TestServiceLogs1": "hello1", + "TestServiceLogs2": "hello2", + } - out, err := d.Cmd("service", "create", "--name", name, "--restart-condition", "none", "busybox", "sh", "-c", "echo hello world") - c.Assert(err, checker.IsNil) - c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") + for name, message := range services { + out, err := d.Cmd("service", "create", "--name", name, "busybox", + "sh", "-c", fmt.Sprintf("echo %s; tail -f /dev/null", message)) + c.Assert(err, checker.IsNil) + c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") + } // make sure task has been deployed. - waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) - - out, err = d.Cmd("service", "logs", name) - fmt.Println(out) - c.Assert(err, checker.IsNil) - c.Assert(out, checker.Contains, "hello world") + waitAndAssert(c, defaultReconciliationTimeout, + d.checkActiveContainerCount, checker.Equals, len(services)) + + for name, message := range services { + out, err := d.Cmd("service", "logs", name) + c.Assert(err, checker.IsNil) + c.Logf("log for %q: %q", name, out) + c.Assert(out, checker.Contains, message) + } } func (s *DockerSwarmSuite) TestServiceLogsFollow(c *check.C) { From 487fad2a2ca96d62264197814e9d35a92bca001d Mon Sep 17 00:00:00 2001 From: erxian Date: Fri, 2 Dec 2016 17:05:03 +0800 Subject: [PATCH 06/68] refine api swagger.yaml towards image create status code Signed-off-by: erxian (cherry picked from commit 15be050fb307d0da1521767a6f53d9b95429396b) Signed-off-by: Victor Vieux --- api/swagger.yaml | 4 ++++ docs/api/v1.18.md | 1 + docs/api/v1.19.md | 1 + docs/api/v1.20.md | 1 + docs/api/v1.21.md | 1 + docs/api/v1.22.md | 1 + docs/api/v1.23.md | 1 + docs/api/v1.24.md | 1 + 8 files changed, 11 insertions(+) diff --git a/api/swagger.yaml b/api/swagger.yaml index 766d6965d473926c6b764c98948d70d6917a8bfd..d159ea9d85cee4c3e88f827f59ad64397e051423 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -4448,6 +4448,10 @@ paths: responses: 200: description: "no error" + 404: + description: "repository does not exist or no read access" + schema: + $ref: "#/definitions/ErrorResponse" 500: description: "server error" schema: diff --git a/docs/api/v1.18.md b/docs/api/v1.18.md index df063c20bcc571383385010580b60ce7004ab9d3..0f127176f4b03879c47c3d4bba6f1580e57b0440 100644 --- a/docs/api/v1.18.md +++ b/docs/api/v1.18.md @@ -1294,6 +1294,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.19.md b/docs/api/v1.19.md index d22c158d899dacf055b900129380a2e858fac682..326679d80f64040d67f1acadf91fc8b672e01610 100644 --- a/docs/api/v1.19.md +++ b/docs/api/v1.19.md @@ -1340,6 +1340,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.20.md b/docs/api/v1.20.md index ab83b3eca164999daff7e9b6cdbaf23f8aaad142..cdd6ee634f4481a0c8b211f73bfd9a81dd023b89 100644 --- a/docs/api/v1.20.md +++ b/docs/api/v1.20.md @@ -1494,6 +1494,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.21.md b/docs/api/v1.21.md index 3ab8ac0c9e3c3ac30333d5a8d0824c63bb4da987..abac14f744fae17ff42fd696d2fb483ab56db45c 100644 --- a/docs/api/v1.21.md +++ b/docs/api/v1.21.md @@ -1587,6 +1587,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.22.md b/docs/api/v1.22.md index 943224cabd3fc17e1e4411144af31cdacdb46cd2..ca7d560ba8c87d6e0d8cf0da8e934dd955e525d9 100644 --- a/docs/api/v1.22.md +++ b/docs/api/v1.22.md @@ -1785,6 +1785,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.23.md b/docs/api/v1.23.md index d8c39b39e1ec521191b3003c01e4fd4ad6704c32..31ff245fdd24c6288af832f9b55e710b8a1fcc1a 100644 --- a/docs/api/v1.23.md +++ b/docs/api/v1.23.md @@ -1821,6 +1821,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error diff --git a/docs/api/v1.24.md b/docs/api/v1.24.md index d05ee7a2de99862ab4697866a7654aae3327904e..e99a7c25f84792b877af7a8337adbf4f57df2464 100644 --- a/docs/api/v1.24.md +++ b/docs/api/v1.24.md @@ -1818,6 +1818,7 @@ a base64-encoded AuthConfig object. **Status codes**: - **200** – no error +- **404** - repository does not exist or no read access - **500** – server error From 62fdac59bdc25cb78b33d8a1e021e65f62bee907 Mon Sep 17 00:00:00 2001 From: Arash Deshmeh Date: Thu, 1 Dec 2016 23:28:51 -0500 Subject: [PATCH 07/68] Print checkpoint id when creating a checkpoint Signed-off-by: Arash Deshmeh (cherry picked from commit f1df2d5a2e563bd38e797e33bce444edd346521c) Signed-off-by: Victor Vieux --- cli/command/checkpoint/create.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/command/checkpoint/create.go b/cli/command/checkpoint/create.go index 2377b5e2e3d17e48b0deb28030c39b99aa7c2893..473a941733b835fb220f7dca1cec19083cd78be8 100644 --- a/cli/command/checkpoint/create.go +++ b/cli/command/checkpoint/create.go @@ -1,6 +1,8 @@ package checkpoint import ( + "fmt" + "golang.org/x/net/context" "github.com/docker/docker/api/types" @@ -51,5 +53,6 @@ func runCreate(dockerCli *command.DockerCli, opts createOptions) error { return err } + fmt.Fprintf(dockerCli.Out(), "%s\n", opts.checkpoint) return nil } From e0d8cfaa7da618ed413871eccc578e7571e569f1 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sun, 4 Dec 2016 11:25:41 -0800 Subject: [PATCH 08/68] Add bash completion for secret management Signed-off-by: Harald Albers (cherry picked from commit 324dd3cfec756109df58128376b888684d8cf521) Signed-off-by: Victor Vieux --- contrib/completion/bash/docker | 113 +++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 63b1b2e653357245b9a63c55c77174dceafb9b8d..a0124684a6b2e4b2fdd46c8c2c4d0ee04c1cf1aa 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -23,6 +23,7 @@ # DOCKER_COMPLETION_SHOW_CONTAINER_IDS # DOCKER_COMPLETION_SHOW_NETWORK_IDS # DOCKER_COMPLETION_SHOW_NODE_IDS +# DOCKER_COMPLETION_SHOW_SECRET_IDS # DOCKER_COMPLETION_SHOW_SERVICE_IDS # "no" - Show names only (default) # "yes" - Show names and ids @@ -311,6 +312,22 @@ __docker_complete_runtimes() { COMPREPLY=( $(compgen -W "$(__docker_runtimes)" -- "$cur") ) } +# __docker_secrets returns a list of all secrets. +# By default, only names of secrets are returned. +# Set DOCKER_COMPLETION_SHOW_SECRET_IDS=yes to also complete IDs of secrets. +__docker_secrets() { + local fields='$2' # default: name only + [ "${DOCKER_COMPLETION_SHOW_SECRET_IDS}" = yes ] && fields='$1,$2' # ID and name + + __docker_q secret ls | awk "NR>1 {print $fields}" +} + +# __docker_complete_secrets applies completion of secrets based on the current value +# of `$cur`. +__docker_complete_secrets() { + COMPREPLY=( $(compgen -W "$(__docker_secrets)" -- "$cur") ) +} + # __docker_stacks returns a list of all stacks. __docker_stacks() { __docker_q stack ls | awk 'NR>1 {print $1}' @@ -2736,6 +2753,7 @@ _docker_service_update() { --mode --name --port + --secret " case "$prev" in @@ -2755,6 +2773,10 @@ _docker_service_update() { COMPREPLY=( $( compgen -W "global replicated" -- "$cur" ) ) return ;; + --secret) + __docker_complete_secrets + return + ;; --group) COMPREPLY=( $(compgen -g -- "$cur") ) return @@ -2779,6 +2801,8 @@ _docker_service_update() { --image --port-add --port-rm + --secret-add + --secret-rm " case "$prev" in @@ -2802,6 +2826,10 @@ _docker_service_update() { __docker_complete_image_repos_and_tags return ;; + --secret-add|--secret-rm) + __docker_complete_secrets + return + ;; esac fi @@ -3329,6 +3357,90 @@ _docker_save() { _docker_image_save } + +_docker_secret() { + local subcommands=" + create + inspect + ls + rm + " + local aliases=" + list + remove + " + __docker_subcommands "$subcommands $aliases" && return + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + COMPREPLY=( $( compgen -W "$subcommands" -- "$cur" ) ) + ;; + esac +} + +_docker_secret_create() { + case "$prev" in + --label|-l) + return + ;; + esac + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help --label -l" -- "$cur" ) ) + ;; + esac +} + +_docker_secret_inspect() { + case "$prev" in + --format|-f) + return + ;; + esac + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--format -f --help" -- "$cur" ) ) + ;; + *) + __docker_complete_secrets + ;; + esac +} + +_docker_secret_list() { + _docker_secret_ls +} + +_docker_secret_ls() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help --quiet -q" -- "$cur" ) ) + ;; + esac +} + +_docker_secret_remove() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_complete_secrets + ;; + esac +} + +_docker_secret_rm() { + _docker_secret_remove +} + + + _docker_search() { local key=$(__docker_map_key_of_current_option '--filter|-f') case "$key" in @@ -3852,6 +3964,7 @@ _docker() { run save search + secret service stack start From fd7aee8fe003c28e53eb684e332786d23e427926 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sun, 4 Dec 2016 12:11:33 -0800 Subject: [PATCH 09/68] Fix bash completion for `docker service create|update Signed-off-by: Harald Albers (cherry picked from commit be5685e4bffc88a6cd3a093ee883c5335758266d) Signed-off-by: Victor Vieux --- contrib/completion/bash/docker | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index a0124684a6b2e4b2fdd46c8c2c4d0ee04c1cf1aa..29e01d6ef29a7e0cf9f8c20f446888447240fe50 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -2874,9 +2874,17 @@ _docker_service_update() { COMPREPLY=( $( compgen -W "$boolean_options $options_with_args" -- "$cur" ) ) ;; *) + local counter=$( __docker_pos_first_nonflag $( __docker_to_alternatives "$options_with_args" ) ) if [ "$subcommand" = "update" ] ; then - __docker_complete_services + if [ $cword -eq $counter ]; then + __docker_complete_services + fi + else + if [ $cword -eq $counter ]; then + __docker_complete_images + fi fi + ;; esac } From 6ce7fb23f5850112b42cb71aeff7cb2d8f414a9c Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sun, 4 Dec 2016 14:13:14 -0800 Subject: [PATCH 10/68] Add bash completion for `docker version --format` Signed-off-by: Harald Albers (cherry picked from commit 3d43c48c1bbb8eaa2be4ec0d6203183b0f55b9d9) Signed-off-by: Victor Vieux --- contrib/completion/bash/docker | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 29e01d6ef29a7e0cf9f8c20f446888447240fe50..b1e146f83431672cc6db98dc95b0eb7527d56439 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -3810,9 +3810,15 @@ _docker_top() { } _docker_version() { + case "$prev" in + --format|-f) + return + ;; + esac + case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--format -f --help" -- "$cur" ) ) ;; esac } From 0f2364f73cb051a102351b00cfc9e7ea545bbd22 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 5 Dec 2016 15:18:36 +0100 Subject: [PATCH 11/68] Handle logging in compose to swarm Logging configuration was completely ignore when deploy a compose file to swarm. This fixes it. Signed-off-by: Vincent Demeester (cherry picked from commit 806cc1e0f815d7c4e4b7bd4fa537e3d4c3535e23) Signed-off-by: Victor Vieux --- api/types/swarm/common.go | 6 ++++++ api/types/swarm/network.go | 6 ------ cli/command/stack/deploy.go | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/api/types/swarm/common.go b/api/types/swarm/common.go index 589e2cfdf268f2bb069c0b7910b60ab120b662eb..64a648bad15a95790ed7db3a1af3b1c57e336235 100644 --- a/api/types/swarm/common.go +++ b/api/types/swarm/common.go @@ -19,3 +19,9 @@ type Annotations struct { Name string `json:",omitempty"` Labels map[string]string `json:",omitempty"` } + +// Driver represents a driver (network, logging). +type Driver struct { + Name string `json:",omitempty"` + Options map[string]string `json:",omitempty"` +} diff --git a/api/types/swarm/network.go b/api/types/swarm/network.go index 2ba53391170b5e0e0d9ea3d4d6e4e32925e9ea3d..5a5e11bdba59a27593651c47c9a8503a2b31e934 100644 --- a/api/types/swarm/network.go +++ b/api/types/swarm/network.go @@ -109,9 +109,3 @@ type IPAMConfig struct { Range string `json:",omitempty"` Gateway string `json:",omitempty"` } - -// Driver represents a network driver. -type Driver struct { - Name string `json:",omitempty"` - Options map[string]string `json:",omitempty"` -} diff --git a/cli/command/stack/deploy.go b/cli/command/stack/deploy.go index e7764f3b8d35217ea57b551256effda9ea1ace30..1f41cb7d89e005973b528517d95fb2641f7a9a79 100644 --- a/cli/command/stack/deploy.go +++ b/cli/command/stack/deploy.go @@ -567,6 +567,14 @@ func convertService( return swarm.ServiceSpec{}, err } + var logDriver *swarm.Driver + if service.Logging != nil { + logDriver = &swarm.Driver{ + Name: service.Logging.Driver, + Options: service.Logging.Options, + } + } + serviceSpec := swarm.ServiceSpec{ Annotations: swarm.Annotations{ Name: name, @@ -589,6 +597,7 @@ func convertService( TTY: service.Tty, OpenStdin: service.StdinOpen, }, + LogDriver: logDriver, Resources: resources, RestartPolicy: restartPolicy, Placement: &swarm.Placement{ From 540758ee99acb5e4cbb6ec0bdf1ad2d081f38fc7 Mon Sep 17 00:00:00 2001 From: John Howard Date: Mon, 5 Dec 2016 09:22:42 -0800 Subject: [PATCH 12/68] Windows: make.ps1 Throw exception on failure Signed-off-by: John Howard (cherry picked from commit 8c22a00b77043db50a1b4837a66dfb27dab3f070) Signed-off-by: Victor Vieux --- hack/make.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/make.ps1 b/hack/make.ps1 index 6084b349206954647e1e4f02a0e8daee50e3e2f2..e9e04781bb9fa67b59f72d371704b80321edab2a 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -394,6 +394,8 @@ Catch [Exception] { Write-Host -ForegroundColor Red " \___ / (____ /__`|____/\___ `>____ `| " Write-Host -ForegroundColor Red " \/ \/ \/ \/ " Write-Host + + Throw $_ } Finally { if ($global:pushed) { Pop-Location } From ad794bba5bbf9af870721432af9ca9dcab7f3343 Mon Sep 17 00:00:00 2001 From: John Howard Date: Mon, 5 Dec 2016 14:01:17 -0800 Subject: [PATCH 13/68] Windows: Dockerfile 2GB clarification Hyper-V Signed-off-by: John Howard (cherry picked from commit f7b4d6544506f882f3168ab6a2ccb75f0ad0794e) Signed-off-by: Victor Vieux --- Dockerfile.windows | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index c6e897b7c872ab193912de67805fa1d8b1bdb55d..3e6fd83418904e0f72897cef437c5e3c739bda7b 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -62,14 +62,17 @@ # >> cd C:\go\src\github.com\docker\docker # # -# 3. Build a docker image with the components required to build the docker binaries from source: +# 3. Build a docker image with the components required to build the docker binaries from source +# by running one of the following: # -# >> docker build -t nativebuildimage -f Dockerfile.windows . +# >> docker build -t nativebuildimage -f Dockerfile.windows . +# >> docker build -t nativebuildimage -f Dockerfile.windows -m 2GB . (if using Hyper-V containers) # # -# 4. Build the docker executable binaries: +# 4. Build the docker executable binaries by running one of the following: # # >> docker run --name binaries nativebuildimage hack\make.ps1 -Binary +# >> docker run --name binaries -m 2GB nativebuildimage hack\make.ps1 -Binary (if using Hyper-V containers) # # # 5. Copy the binaries out of the container, replacing HostPath with an appropriate destination @@ -96,10 +99,11 @@ # The validation tests can either run in a container, or directly on the host. To run in a -# container, ensure you have created the nativebuildimage above. Then run the following -# from an (elevated) Windows PowerShell prompt: +# container, ensure you have created the nativebuildimage above. Then run one of the +# following from an (elevated) Windows PowerShell prompt: # # >> docker run --rm nativebuildimage hack\make.ps1 -DCO -PkgImports -GoFormat +# >> docker run --rm -m 2GB nativebuildimage hack\make.ps1 -DCO -PkgImports -GoFormat (if using Hyper-V containers) # To run the validation tests on the host, from the root of the repository, run the # following from a Windows PowerShell prompt (elevation is not required): (Note Go @@ -110,20 +114,21 @@ # ----------------------------------------------------------------------------------------- -# To run unit tests, ensure you have created the nativebuildimage above. Then run the -# following from an (elevated) Windows PowerShell prompt: +# To run unit tests, ensure you have created the nativebuildimage above. Then run one of +# the following from an (elevated) Windows PowerShell prompt: # # >> docker run --rm nativebuildimage hack\make.ps1 -TestUnit +# >> docker run --rm -m 2GB nativebuildimage hack\make.ps1 -TestUnit (if using Hyper-V containers) # ----------------------------------------------------------------------------------------- # To run all tests and binary build, ensure you have created the nativebuildimage above. Then -# run the following from an (elevated) Windows PowerShell prompt: +# run one of the following from an (elevated) Windows PowerShell prompt: # # >> docker run nativebuildimage hack\make.ps1 -All - +# >> docker run -m 2GB nativebuildimage hack\make.ps1 -All (if using Hyper-V containers) # ----------------------------------------------------------------------------------------- From 51b83ae8fb9405abcecd6647e0339995752928a0 Mon Sep 17 00:00:00 2001 From: allencloud Date: Tue, 6 Dec 2016 14:36:50 +0800 Subject: [PATCH 14/68] add 403 for endpoint network create Signed-off-by: allencloud (cherry picked from commit 0d21e24b9fb90ea804ba37af303de7547743ba2b) Signed-off-by: Victor Vieux --- api/swagger.yaml | 4 ++++ docs/api/v1.24.md | 1 + 2 files changed, 5 insertions(+) diff --git a/api/swagger.yaml b/api/swagger.yaml index d159ea9d85cee4c3e88f827f59ad64397e051423..13699db2c89037051aa5379d0588f431cd342f6b 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -6133,6 +6133,10 @@ paths: example: Id: "22be93d5babb089c5aab8dbc369042fad48ff791584ca2da2100db837a1c7c30" Warning: "" + 403: + description: "operation not supported for pre-defined networks" + schema: + $ref: "#/definitions/ErrorResponse" 404: description: "plugin not found" schema: diff --git a/docs/api/v1.24.md b/docs/api/v1.24.md index e99a7c25f84792b877af7a8337adbf4f57df2464..85c7e83acc743e3f71217e16d2d100fcd5bedd0b 100644 --- a/docs/api/v1.24.md +++ b/docs/api/v1.24.md @@ -3320,6 +3320,7 @@ Content-Type: application/json **Status codes**: - **201** - no error +- **403** - operation not supported for pre-defined networks - **404** - plugin not found - **500** - server error From 0383913b65cbfc20de861c24400c1ccc2dfbcef6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 6 Dec 2016 08:44:02 -0800 Subject: [PATCH 15/68] Add bash completion for `docker network create --attachable` Signed-off-by: Harald Albers (cherry picked from commit 8d2e789bbc59efa75d242457dba4b8651a1f9ace) Signed-off-by: Victor Vieux --- contrib/completion/bash/docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index b1e146f83431672cc6db98dc95b0eb7527d56439..efaffdc509993fcc3f97297cb453ac5d01b1fdd2 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -2442,7 +2442,7 @@ _docker_network_create() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--aux-address --driver -d --gateway --help --internal --ip-range --ipam-driver --ipam-opt --ipv6 --label --opt -o --subnet" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--attachable --aux-address --driver -d --gateway --help --internal --ip-range --ipam-driver --ipam-opt --ipv6 --label --opt -o --subnet" -- "$cur" ) ) ;; esac } From bec385cff03a663cc6d9c5e1b32199a666bf0aa7 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 6 Dec 2016 10:29:48 -0800 Subject: [PATCH 16/68] Fix `docker inspect ` issue on Windows This fix tries to address the issue raised on 29185 where `docker inspect ` on Windows will return: ``` Error response from daemon: plugins are not supported on this platform ``` The reason was that in case `--type` is not specified, `docker inspect` will iterate through different types `container`, `image`, `network`, `plugin` etc. The `plugin` object is the last type to check. However, as `plugin` is not supported on Windows yet, the error message is not very informative for `plugins are not supported on this platform`. This fix tries to fix the issue by return a `not found` error on unsupported platforms as well. An integration test has been added to cover the changes for Windows/Linux. This fix fixes 29185. Signed-off-by: Yong Tang (cherry picked from commit 88fcdb0a825da040ef2b1f9c191af480f0f2cc90) Signed-off-by: Victor Vieux --- integration-cli/docker_cli_inspect_test.go | 9 +++++++++ plugin/backend_unsupported.go | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/integration-cli/docker_cli_inspect_test.go b/integration-cli/docker_cli_inspect_test.go index 5d79ad1d857541e4bd5c3b1fe00c8e519444f5e7..2f2a918f0f98622d55819b73874955b72a772e15 100644 --- a/integration-cli/docker_cli_inspect_test.go +++ b/integration-cli/docker_cli_inspect_test.go @@ -447,3 +447,12 @@ func (s *DockerSuite) TestInspectPlugin(c *check.C) { c.Assert(err, checker.IsNil) c.Assert(out, checker.Contains, pNameWithTag) } + +// Test case for 29185 +func (s *DockerSuite) TestInspectUnknownObject(c *check.C) { + // This test should work on both Windows and Linux + out, _, err := dockerCmdWithError("inspect", "foobar") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "Error: No such object: foobar") + c.Assert(err.Error(), checker.Contains, "Error: No such object: foobar") +} diff --git a/plugin/backend_unsupported.go b/plugin/backend_unsupported.go index 0e07cd679ae8b7cc5bc0764e04590223c3c6eb13..2d4b365fafed5423af0403a8d41b937b723477a3 100644 --- a/plugin/backend_unsupported.go +++ b/plugin/backend_unsupported.go @@ -4,6 +4,7 @@ package plugin import ( "errors" + "fmt" "io" "net/http" @@ -24,8 +25,11 @@ func (pm *Manager) Enable(name string, config *types.PluginEnableConfig) error { } // Inspect examines a plugin config -func (pm *Manager) Inspect(name string) (tp types.Plugin, err error) { - return tp, errNotSupported +func (pm *Manager) Inspect(refOrID string) (tp types.Plugin, err error) { + // Even though plugin is not supported, we still want to return `not found` + // error so that `docker inspect` (without `--type` specified) returns correct + // `not found` message + return tp, fmt.Errorf("no such plugin name or ID associated with %q", refOrID) } // Privileges pulls a plugin config and computes the privileges required to install it. From c9ec321e569459b9ada693d71983d5ea8bd9455e Mon Sep 17 00:00:00 2001 From: Lei Jitang Date: Wed, 7 Dec 2016 01:37:08 -0500 Subject: [PATCH 17/68] fix #29199, reset container if container start failed Signed-off-by: Lei Jitang (cherry picked from commit e806821b53f9ca1f1a3d933e7bbfe04a5566a3bf) Signed-off-by: Victor Vieux --- daemon/start.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/start.go b/daemon/start.go index 1a5b0e75592eae91ef7d3b2d4f3db5d0fa045034..6c94fd5482d0bcde0f4d1c469e7e4a416cf08be0 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -119,6 +119,9 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint container.SetExitCode(128) } container.ToDisk() + + container.Reset(false) + daemon.Cleanup(container) // if containers AutoRemove flag is set, remove it after clean up if container.HostConfig.AutoRemove { @@ -187,8 +190,6 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint container.SetExitCode(127) } - container.Reset(false) - return fmt.Errorf("%s", errDesc) } From 52b88ccc23849e19668671035aa5c8975aeac150 Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Wed, 7 Dec 2016 15:06:16 +0800 Subject: [PATCH 18/68] modify URLs for bind docker in docs/api Signed-off-by: lixiaobing10051267 (cherry picked from commit 9c76fb253ecc53cc54d196a4460d8e1d91c1683d) Signed-off-by: Victor Vieux --- docs/api/v1.18.md | 2 +- docs/api/v1.19.md | 2 +- docs/api/v1.20.md | 2 +- docs/api/v1.21.md | 2 +- docs/api/v1.22.md | 2 +- docs/api/v1.23.md | 2 +- docs/api/v1.24.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/api/v1.18.md b/docs/api/v1.18.md index 0f127176f4b03879c47c3d4bba6f1580e57b0440..513415fd7acbd543a098b03e8ddcfc55d9fc69dd 100644 --- a/docs/api/v1.18.md +++ b/docs/api/v1.18.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST, but for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `STDOUT`, `STDIN` and `STDERR`. diff --git a/docs/api/v1.19.md b/docs/api/v1.19.md index 326679d80f64040d67f1acadf91fc8b672e01610..08f661d3c58c6bd29dc6c56ef12430c897f3eea6 100644 --- a/docs/api/v1.19.md +++ b/docs/api/v1.19.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. diff --git a/docs/api/v1.20.md b/docs/api/v1.20.md index cdd6ee634f4481a0c8b211f73bfd9a81dd023b89..2ce535a693e722988fef49802fd6d9b3a7bb30ac 100644 --- a/docs/api/v1.20.md +++ b/docs/api/v1.20.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. diff --git a/docs/api/v1.21.md b/docs/api/v1.21.md index abac14f744fae17ff42fd696d2fb483ab56db45c..0dca325ba04dccc4031f541a07d37879e1a588dc 100644 --- a/docs/api/v1.21.md +++ b/docs/api/v1.21.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. diff --git a/docs/api/v1.22.md b/docs/api/v1.22.md index ca7d560ba8c87d6e0d8cf0da8e934dd955e525d9..8f5f08bb33d19265ff74f4d705e726b5efef7d76 100644 --- a/docs/api/v1.22.md +++ b/docs/api/v1.22.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. diff --git a/docs/api/v1.23.md b/docs/api/v1.23.md index 31ff245fdd24c6288af832f9b55e710b8a1fcc1a..34250f2f2a5191bfecc9211d7e88961b3e121e2c 100644 --- a/docs/api/v1.23.md +++ b/docs/api/v1.23.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. diff --git a/docs/api/v1.24.md b/docs/api/v1.24.md index 85c7e83acc743e3f71217e16d2d100fcd5bedd0b..81479c083e4d19fa9a2f0f3c2f953ef9f368d188 100644 --- a/docs/api/v1.24.md +++ b/docs/api/v1.24.md @@ -19,7 +19,7 @@ redirect_from: ## 1. Brief introduction - The daemon listens on `unix:///var/run/docker.sock` but you can - [Bind Docker to another host/port or a Unix socket](../commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). + [Bind Docker to another host/port or a Unix socket](../reference/commandline/dockerd.md#bind-docker-to-another-host-port-or-a-unix-socket). - The API tends to be REST. However, for some complex commands, like `attach` or `pull`, the HTTP connection is hijacked to transport `stdout`, `stdin` and `stderr`. From 78f2dc90bf8c5f2b4244b4a1ea522d460c5db74b Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Wed, 7 Dec 2016 08:09:42 +0100 Subject: [PATCH 19/68] Add zsh completion for 'docker network create --attachable' Signed-off-by: Steve Durrheimer (cherry picked from commit bce11a29f822e18ef2bac28f147b5506f785e4cc) Signed-off-by: Victor Vieux --- contrib/completion/zsh/_docker | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 5dfd1ef64c9ac44c68ee63b7990de0c8546f32b2..3afcb8977c51009aa764ae09b3352def66a51069 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -1160,6 +1160,7 @@ __docker_network_subcommand() { (create) _arguments $(__docker_arguments) -A '-*' \ $opts_help \ + "($help)--attachable[Enable manual container attachment]" \ "($help)*--aux-address[Auxiliary IPv4 or IPv6 addresses used by network driver]:key=IP: " \ "($help -d --driver)"{-d=,--driver=}"[Driver to manage the Network]:driver:(null host bridge overlay)" \ "($help)*--gateway=[IPv4 or IPv6 Gateway for the master subnet]:IP: " \ From 6c8bd560076526d92fd303acfefe0c5b550885ff Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Fri, 2 Dec 2016 19:56:27 +0100 Subject: [PATCH 20/68] hack/make.sh: fix BUILDTIME Signed-off-by: Antonio Murdaca (cherry picked from commit 7b1f77dcbc4c7cda754613f424f937056d3206ec) Signed-off-by: Victor Vieux --- hack/make.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hack/make.sh b/hack/make.sh index aaf181291df322fccc7a071c99eb7d87e0566a8e..f0e482fedabb3b1b0634aee12611bde6418dc6b1 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -69,6 +69,7 @@ DEFAULT_BUNDLES=( ) VERSION=$(< ./VERSION) +! BUILDTIME=$(date --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/') if command -v git &> /dev/null && [ -d .git ] && git rev-parse &> /dev/null; then GITCOMMIT=$(git rev-parse --short HEAD) if [ -n "$(git status --porcelain --untracked-files=no)" ]; then @@ -82,11 +83,6 @@ if command -v git &> /dev/null && [ -d .git ] && git rev-parse &> /dev/null; the git status --porcelain --untracked-files=no echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" fi - ! BUILDTIME=$(date --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/') &> /dev/null - if [ -z $BUILDTIME ]; then - # If using bash 3.1 which doesn't support --rfc-3389, eg Windows CI - BUILDTIME=$(date -u) - fi elif [ "$DOCKER_GITCOMMIT" ]; then GITCOMMIT="$DOCKER_GITCOMMIT" else From 90a24f7bfde2f4facf5abca0163918d4cc88ac96 Mon Sep 17 00:00:00 2001 From: yuexiao-wang Date: Fri, 2 Dec 2016 03:32:04 +0800 Subject: [PATCH 21/68] Optimize the log info for client test Signed-off-by: yuexiao-wang (cherry picked from commit 40b8ff62431d2005e5801b88dbe1685e89baafe5) Signed-off-by: Victor Vieux --- client/container_copy_test.go | 8 ++++---- client/image_search_test.go | 8 ++++---- client/plugin_push_test.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/container_copy_test.go b/client/container_copy_test.go index 7eded611fd7f93e4a44546376e3a430ab81e06c3..706a20c8189fd0503b0ed5218a6f433ef08031d2 100644 --- a/client/container_copy_test.go +++ b/client/container_copy_test.go @@ -78,10 +78,10 @@ func TestContainerStatPath(t *testing.T) { t.Fatal(err) } if stat.Name != "name" { - t.Fatalf("expected container path stat name to be 'name', was '%s'", stat.Name) + t.Fatalf("expected container path stat name to be 'name', got '%s'", stat.Name) } if stat.Mode != 0700 { - t.Fatalf("expected container path stat mode to be 0700, was '%v'", stat.Mode) + t.Fatalf("expected container path stat mode to be 0700, got '%v'", stat.Mode) } } @@ -226,10 +226,10 @@ func TestCopyFromContainer(t *testing.T) { t.Fatal(err) } if stat.Name != "name" { - t.Fatalf("expected container path stat name to be 'name', was '%s'", stat.Name) + t.Fatalf("expected container path stat name to be 'name', got '%s'", stat.Name) } if stat.Mode != 0700 { - t.Fatalf("expected container path stat mode to be 0700, was '%v'", stat.Mode) + t.Fatalf("expected container path stat mode to be 0700, got '%v'", stat.Mode) } content, err := ioutil.ReadAll(r) if err != nil { diff --git a/client/image_search_test.go b/client/image_search_test.go index e46d86437f77334ad0543a171c78286ab4bdd680..108bd967444e8ad754c74af9722f5e4c55c1a0cd 100644 --- a/client/image_search_test.go +++ b/client/image_search_test.go @@ -81,12 +81,12 @@ func TestImageSearchWithPrivilegedFuncNoError(t *testing.T) { }, nil } if auth != "IAmValid" { - return nil, fmt.Errorf("Invalid auth header : expected %s, got %s", "IAmValid", auth) + return nil, fmt.Errorf("Invalid auth header : expected 'IAmValid', got %s", auth) } query := req.URL.Query() term := query.Get("term") if term != "some-image" { - return nil, fmt.Errorf("tag not set in URL query properly. Expected '%s', got %s", "some-image", term) + return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) } content, err := json.Marshal([]registry.SearchResult{ { @@ -113,7 +113,7 @@ func TestImageSearchWithPrivilegedFuncNoError(t *testing.T) { t.Fatal(err) } if len(results) != 1 { - t.Fatalf("expected a result, got %v", results) + t.Fatalf("expected 1 result, got %v", results) } } @@ -133,7 +133,7 @@ func TestImageSearchWithoutErrors(t *testing.T) { query := req.URL.Query() term := query.Get("term") if term != "some-image" { - return nil, fmt.Errorf("tag not set in URL query properly. Expected '%s', got %s", "some-image", term) + return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) } filters := query.Get("filters") if filters != expectedFilters { diff --git a/client/plugin_push_test.go b/client/plugin_push_test.go index efdbdc6db18ba4e3bf09b593b2c4ff16ad7ee3ba..7b8eb865d620e506e27268a1d4c0cdc3a90473fe 100644 --- a/client/plugin_push_test.go +++ b/client/plugin_push_test.go @@ -35,7 +35,7 @@ func TestPluginPush(t *testing.T) { } auth := req.Header.Get("X-Registry-Auth") if auth != "authtoken" { - return nil, fmt.Errorf("Invalid auth header : expected %s, got %s", "authtoken", auth) + return nil, fmt.Errorf("Invalid auth header : expected 'authtoken', got %s", auth) } return &http.Response{ StatusCode: http.StatusOK, From 674cf8233daf315cd64bb2db26e5881891a01351 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 5 Dec 2016 17:00:36 +0200 Subject: [PATCH 22/68] api/types/container,client: gofmt Signed-off-by: Cristian Staretu (cherry picked from commit c1ce63b17bde361107900d7b58da6d13fd309734) Signed-off-by: Victor Vieux --- client/image_search_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/image_search_test.go b/client/image_search_test.go index 108bd967444e8ad754c74af9722f5e4c55c1a0cd..b17bbd83436d4b3bca1767b1f46490c8c539ae8d 100644 --- a/client/image_search_test.go +++ b/client/image_search_test.go @@ -86,7 +86,7 @@ func TestImageSearchWithPrivilegedFuncNoError(t *testing.T) { query := req.URL.Query() term := query.Get("term") if term != "some-image" { - return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) + return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) } content, err := json.Marshal([]registry.SearchResult{ { @@ -133,7 +133,7 @@ func TestImageSearchWithoutErrors(t *testing.T) { query := req.URL.Query() term := query.Get("term") if term != "some-image" { - return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) + return nil, fmt.Errorf("term not set in URL query properly. Expected 'some-image', got %s", term) } filters := query.Get("filters") if filters != expectedFilters { From 9ecbaa77ae8e16b5d481c3c3e666fdae8a77b916 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 15 Nov 2016 17:31:54 -0800 Subject: [PATCH 23/68] Update vendor distribution Distribution client change for class in resource Signed-off-by: Derek McGowan (github: dmcgowan) (cherry picked from commit d1f5e0f7a68d8a43931b834ce342830956bdac32) Signed-off-by: Victor Vieux --- vendor.conf | 2 +- .../docker/distribution/registry/client/auth/session.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vendor.conf b/vendor.conf index 5d272a83e17be2bfa541a82d333c4a7a34abb291..d59b72d9c3cf11cdf6ebfaffdb018cdda7c7857b 100644 --- a/vendor.conf +++ b/vendor.conf @@ -44,7 +44,7 @@ github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages -github.com/docker/distribution d22e09a6686c32be8c17b684b639da4b90efe320 +github.com/docker/distribution a6bf3dd064f15598166bca2d66a9962a9555139e github.com/vbatts/tar-split v0.10.1 # get go-zfs packages diff --git a/vendor/github.com/docker/distribution/registry/client/auth/session.go b/vendor/github.com/docker/distribution/registry/client/auth/session.go index ffc3384b1993ebfb5b1cac006f7d719ad2eaea2a..d6d884ffd1dd9601aeb9648ba86134bab20c97b8 100644 --- a/vendor/github.com/docker/distribution/registry/client/auth/session.go +++ b/vendor/github.com/docker/distribution/registry/client/auth/session.go @@ -147,13 +147,18 @@ type Scope interface { // to a repository. type RepositoryScope struct { Repository string + Class string Actions []string } // String returns the string representation of the repository // using the scope grammar func (rs RepositoryScope) String() string { - return fmt.Sprintf("repository:%s:%s", rs.Repository, strings.Join(rs.Actions, ",")) + repoType := "repository" + if rs.Class != "" { + repoType = fmt.Sprintf("%s(%s)", repoType, rs.Class) + } + return fmt.Sprintf("%s:%s:%s", repoType, rs.Repository, strings.Join(rs.Actions, ",")) } // RegistryScope represents a token scope for access From cca0132fcfd1957baff7d34925e83dc3f4a67d52 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 15 Nov 2016 15:06:48 -0800 Subject: [PATCH 24/68] Add class to repository scope Expose registry error translation for plugin distribution Signed-off-by: Derek McGowan (github: dmcgowan) (cherry picked from commit a12b466183e03621bc9e1c1e4deab6db8ec93f0a) Signed-off-by: Victor Vieux --- distribution/errors.go | 12 ++++++++---- distribution/pull.go | 4 ++-- distribution/registry.go | 19 ++++++++++++------- integration-cli/docker_cli_pull_test.go | 2 +- plugin/distribution/pull.go | 4 ++-- plugin/distribution/push.go | 1 + registry/config.go | 6 +++++- registry/types.go | 3 +++ 8 files changed, 34 insertions(+), 17 deletions(-) diff --git a/distribution/errors.go b/distribution/errors.go index 7f9e20f279d65ed283b524599d0f2c94813f3120..b8cf9fb9e8034330d6195cdbfdec5f2bffaf0681 100644 --- a/distribution/errors.go +++ b/distribution/errors.go @@ -60,21 +60,25 @@ func shouldV2Fallback(err errcode.Error) bool { return false } -func translatePullError(err error, ref reference.Named) error { +// TranslatePullError is used to convert an error from a registry pull +// operation to an error representing the entire pull operation. Any error +// information which is not used by the returned error gets output to +// log at info level. +func TranslatePullError(err error, ref reference.Named) error { switch v := err.(type) { case errcode.Errors: if len(v) != 0 { for _, extra := range v[1:] { logrus.Infof("Ignoring extra error returned from registry: %v", extra) } - return translatePullError(v[0], ref) + return TranslatePullError(v[0], ref) } case errcode.Error: var newErr error switch v.Code { case errcode.ErrorCodeDenied: // ErrorCodeDenied is used when access to the repository was denied - newErr = errors.Errorf("repository %s not found: does not exist or no read access", ref.Name()) + newErr = errors.Errorf("repository %s not found: does not exist or no pull access", ref.Name()) case v2.ErrorCodeManifestUnknown: newErr = errors.Errorf("manifest for %s not found", ref.String()) case v2.ErrorCodeNameUnknown: @@ -85,7 +89,7 @@ func translatePullError(err error, ref reference.Named) error { return newErr } case xfer.DoNotRetry: - return translatePullError(v.Err, ref) + return TranslatePullError(v.Err, ref) } return err diff --git a/distribution/pull.go b/distribution/pull.go index 5307d4ccc977cefbf44de1a278bad1003353bb4d..b631788b49431a6e1ceb78bc60bfe5809025ad4e 100644 --- a/distribution/pull.go +++ b/distribution/pull.go @@ -168,7 +168,7 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo continue } logrus.Errorf("Not continuing with pull after error: %v", err) - return translatePullError(err, ref) + return TranslatePullError(err, ref) } imagePullConfig.ImageEventLogger(ref.String(), repoInfo.Name(), "pull") @@ -179,7 +179,7 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo lastErr = fmt.Errorf("no endpoints found for %s", ref.String()) } - return translatePullError(lastErr, ref) + return TranslatePullError(lastErr, ref) } // writeStatus writes a status message to out. If layersDownloaded is true, the diff --git a/distribution/registry.go b/distribution/registry.go index d3c991e875929fb61378f549ceec6fab2948fd14..4c3513046d4953186ed714e1747de6b0d741381d 100644 --- a/distribution/registry.go +++ b/distribution/registry.go @@ -70,17 +70,22 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken} modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler)) } else { + scope := auth.RepositoryScope{ + Repository: repoName, + Actions: actions, + } + + // Keep image repositories blank for scope compatibility + if repoInfo.Class != "image" { + scope.Class = repoInfo.Class + } + creds := registry.NewStaticCredentialStore(authConfig) tokenHandlerOptions := auth.TokenHandlerOptions{ Transport: authTransport, Credentials: creds, - Scopes: []auth.Scope{ - auth.RepositoryScope{ - Repository: repoName, - Actions: actions, - }, - }, - ClientID: registry.AuthClientID, + Scopes: []auth.Scope{scope}, + ClientID: registry.AuthClientID, } tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions) basicHandler := auth.NewBasicHandler(creds) diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index c89adae0c69c12c5d99d90e9dec7157b5955b488..a0118a8e95036c2624e6a8a698c1f8176afea4e0 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -98,7 +98,7 @@ func (s *DockerHubPullSuite) TestPullNonExistingImage(c *check.C) { for record := range recordChan { if len(record.option) == 0 { c.Assert(record.err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", record.out)) - c.Assert(record.out, checker.Contains, fmt.Sprintf("repository %s not found: does not exist or no read access", record.e.repo), check.Commentf("expected image not found error messages")) + c.Assert(record.out, checker.Contains, fmt.Sprintf("repository %s not found: does not exist or no pull access", record.e.repo), check.Commentf("expected image not found error messages")) } else { // pull -a on a nonexistent registry should fall back as well c.Assert(record.err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", record.out)) diff --git a/plugin/distribution/pull.go b/plugin/distribution/pull.go index dba750f2a5e672edf0627b58b96d3ec28b5bb4cc..4e8992cc2ea5e3c5fb5f0662c67cd9b782143144 100644 --- a/plugin/distribution/pull.go +++ b/plugin/distribution/pull.go @@ -85,6 +85,7 @@ func Pull(ref reference.Named, rs registry.Service, metaheader http.Header, auth logrus.Debugf("pull.go: error in ResolveRepository: %v", err) return nil, err } + repoInfo.Class = "plugin" if err := dockerdist.ValidateRepoName(repoInfo.Name()); err != nil { logrus.Debugf("pull.go: error in ValidateRepoName: %v", err) @@ -138,9 +139,8 @@ func Pull(ref reference.Named, rs registry.Service, metaheader http.Header, auth } manifest, err := msv.Get(context.Background(), "", distribution.WithTag(tag)) if err != nil { - // TODO: change 401 to 404 logrus.Debugf("pull.go: error in msv.Get(): %v", err) - return nil, err + return nil, dockerdist.TranslatePullError(err, repoInfo) } _, pl, err := manifest.Payload() diff --git a/plugin/distribution/push.go b/plugin/distribution/push.go index 104b684d54f2c06e9b6fb44ee33a70e6c7f5de8b..86caadbc1e6abfea87b165c76e9c22016fb82cb0 100644 --- a/plugin/distribution/push.go +++ b/plugin/distribution/push.go @@ -27,6 +27,7 @@ func Push(name string, rs registry.Service, metaHeader http.Header, authConfig * if err != nil { return "", err } + repoInfo.Class = "plugin" if err := dockerdist.ValidateRepoName(repoInfo.Name()); err != nil { return "", err diff --git a/registry/config.go b/registry/config.go index b53c11bdef1c116640dec4556506d2169e837f8b..9a4f6a92515ce3be41a345a088a33b15621c4c47 100644 --- a/registry/config.go +++ b/registry/config.go @@ -280,7 +280,11 @@ func newRepositoryInfo(config *serviceConfig, name reference.Named) (*Repository return nil, err } official := !strings.ContainsRune(name.Name(), '/') - return &RepositoryInfo{name, index, official}, nil + return &RepositoryInfo{ + Named: name, + Index: index, + Official: official, + }, nil } // ParseRepositoryInfo performs the breakdown of a repository name into a RepositoryInfo, but diff --git a/registry/types.go b/registry/types.go index 479d22a7a6a48c84403eace305ad208fff21e67f..49c123a3e2cfcba3ce48509b223769a83600baeb 100644 --- a/registry/types.go +++ b/registry/types.go @@ -67,4 +67,7 @@ type RepositoryInfo struct { // If the registry is official, and the normalized name does not // contain a '/' (e.g. "foo"), then it is considered an official repo. Official bool + // Class represents the class of the repository, such as "plugin" + // or "image". + Class string } From be7d576f5db1b96b92d801a811cf161ebca945a7 Mon Sep 17 00:00:00 2001 From: Xianglin Gao Date: Tue, 29 Nov 2016 15:48:38 +0800 Subject: [PATCH 25/68] fix apparmor load profile Signed-off-by: Xianglin Gao (cherry picked from commit 2ab8f2e389b4ae90d0cec6555ea5708ceca1cc3c) Signed-off-by: Victor Vieux --- pkg/aaparser/aaparser.go | 3 +-- profiles/apparmor/template.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/aaparser/aaparser.go b/pkg/aaparser/aaparser.go index 4eee2a16798eb3cfb2774ea23fd93ca682de612b..ffcc5647a9f86d4afe8d65aa03d496261962a3db 100644 --- a/pkg/aaparser/aaparser.go +++ b/pkg/aaparser/aaparser.go @@ -4,7 +4,6 @@ package aaparser import ( "fmt" "os/exec" - "path/filepath" "strconv" "strings" ) @@ -26,7 +25,7 @@ func GetVersion() (int, error) { // LoadProfile runs `apparmor_parser -r` on a specified apparmor profile to // replace the profile. func LoadProfile(profilePath string) error { - _, err := cmd("", "-r", filepath.Dir(profilePath)) + _, err := cmd("", "-r", profilePath) if err != nil { return err } diff --git a/profiles/apparmor/template.go b/profiles/apparmor/template.go index dd9da97216f67edd9a7dccb3f8656a03a48827f2..c5ea4584de6b51c0bd7da08db901c2812f48c14b 100644 --- a/profiles/apparmor/template.go +++ b/profiles/apparmor/template.go @@ -40,7 +40,7 @@ profile {{.Name}} flags=(attach_disconnected,mediate_deleted) { {{if ge .Version 208095}} # suppress ptrace denials when using 'docker ps' or using 'ps' inside a container - ptrace (trace,read) peer=docker-default, + ptrace (trace,read) peer={{.Name}}, {{end}} } ` From c84f43d63f0066f95a949fb592d5e2cadf8d3d83 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 5 Dec 2016 10:13:07 +0000 Subject: [PATCH 26/68] Caution against the use of CONFIG_LEGACY_VSYSCALL_NATIVE It provides an ASLR-bypassing target with usable ROP gadgets. Signed-off-by: Ian Campbell (cherry picked from commit 49dcce7ba0a067b62d7791a0525f23b80cd7ad24) Signed-off-by: Victor Vieux --- contrib/check-config.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/check-config.sh b/contrib/check-config.sh index a6029e310e4b7bb026ec23ac72a7e96f0c2a6836..d07e4ce3682dfe95de382ece1b6d437d98ab45df 100755 --- a/contrib/check-config.sh +++ b/contrib/check-config.sh @@ -224,7 +224,8 @@ echo 'Optional Features:' } { if is_set LEGACY_VSYSCALL_NATIVE; then - echo -n "- "; wrap_good "CONFIG_LEGACY_VSYSCALL_NATIVE" 'enabled' + echo -n "- "; wrap_bad "CONFIG_LEGACY_VSYSCALL_NATIVE" 'enabled' + echo " $(wrap_color '(dangerous, provides an ASLR-bypassing target with usable ROP gadgets.)' bold black)" elif is_set LEGACY_VSYSCALL_EMULATE; then echo -n "- "; wrap_good "CONFIG_LEGACY_VSYSCALL_EMULATE" 'enabled' elif is_set LEGACY_VSYSCALL_NONE; then From f081b22a4af0010c3868be8f0f92458b7c37c485 Mon Sep 17 00:00:00 2001 From: Evan Hazlett Date: Mon, 5 Dec 2016 10:58:53 -0500 Subject: [PATCH 27/68] add headers when using exec Signed-off-by: Evan Hazlett ensure headers are properly sanitized Signed-off-by: Evan Hazlett (cherry picked from commit f86db80b5ff3250a98482b4dc9ff69effbbf2390) Signed-off-by: Victor Vieux --- api/server/router/container/exec.go | 10 ++++++++-- integration-cli/docker_api_exec_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/server/router/container/exec.go b/api/server/router/container/exec.go index 8d49de7dac8057977dced1fd0de00ef7cf5b7ff0..1134a0e797a18d865d841959a93bec27677e6ad9 100644 --- a/api/server/router/container/exec.go +++ b/api/server/router/container/exec.go @@ -92,11 +92,17 @@ func (s *containerRouter) postContainerExecStart(ctx context.Context, w http.Res defer httputils.CloseStreams(inStream, outStream) if _, ok := r.Header["Upgrade"]; ok { - fmt.Fprintf(outStream, "HTTP/1.1 101 UPGRADED\r\nContent-Type: application/vnd.docker.raw-stream\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\n") + fmt.Fprint(outStream, "HTTP/1.1 101 UPGRADED\r\nContent-Type: application/vnd.docker.raw-stream\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n") } else { - fmt.Fprintf(outStream, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") + fmt.Fprint(outStream, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n") } + // copy headers that were removed as part of hijack + if err := w.Header().WriteSubset(outStream, nil); err != nil { + return err + } + fmt.Fprint(outStream, "\r\n") + stdin = inStream stdout = outStream if !execStartCheck.Tty { diff --git a/integration-cli/docker_api_exec_test.go b/integration-cli/docker_api_exec_test.go index e792c8f512d3c0ece5e6c914e360521f75c881d1..716e9ac68fb476804cd556c800670cd5e264ba79 100644 --- a/integration-cli/docker_api_exec_test.go +++ b/integration-cli/docker_api_exec_test.go @@ -89,6 +89,16 @@ func (s *DockerSuite) TestExecAPIStart(c *check.C) { startExec(c, id, http.StatusOK) } +func (s *DockerSuite) TestExecAPIStartEnsureHeaders(c *check.C) { + testRequires(c, DaemonIsLinux) + dockerCmd(c, "run", "-d", "--name", "test", "busybox", "top") + + id := createExec(c, "test") + resp, _, err := sockRequestRaw("POST", fmt.Sprintf("/exec/%s/start", id), strings.NewReader(`{"Detach": true}`), "application/json") + c.Assert(err, checker.IsNil) + c.Assert(resp.Header.Get("Server"), checker.Not(checker.Equals), "") +} + func (s *DockerSuite) TestExecAPIStartBackwardsCompatible(c *check.C) { testRequires(c, DaemonIsLinux) // Windows only supports 1.25 or later runSleepingContainer(c, "-d", "--name", "test") From 213ee20706b0c8615912091a47a3168fff219445 Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Thu, 1 Dec 2016 11:36:56 -0800 Subject: [PATCH 28/68] Make v2/Plugin accesses safe. v2/Plugin struct had fields that were - purely used by the manager. - unsafely exposed without proper locking. This change fixes this, by moving relevant fields to the manager as well as making remaining fields as private and providing proper accessors for them. Signed-off-by: Anusha Ragunathan (cherry picked from commit b35490a8ba2ad70a585c1ba8109b6d87aece8daa) Signed-off-by: Victor Vieux --- plugin/backend_linux.go | 23 +++++----- plugin/manager.go | 37 +++++++++++----- plugin/manager_linux.go | 45 +++++++++++--------- plugin/manager_solaris.go | 4 +- plugin/manager_windows.go | 4 +- plugin/store/store.go | 4 +- plugin/v2/plugin.go | 90 ++++++++++++++++++++++++++++----------- 7 files changed, 133 insertions(+), 74 deletions(-) diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go index 4b4fd11a65e4a6d56015aa2754e2c8de9a395c57..f9396626c3201f3094719f9a1f953dfa0810ea8e 100644 --- a/plugin/backend_linux.go +++ b/plugin/backend_linux.go @@ -37,7 +37,11 @@ func (pm *Manager) Disable(name string) error { if err != nil { return err } - if err := pm.disable(p); err != nil { + pm.mu.RLock() + c := pm.cMap[p] + pm.mu.RUnlock() + + if err := pm.disable(p, c); err != nil { return err } pm.pluginEventLogger(p.GetID(), name, "disable") @@ -46,14 +50,13 @@ func (pm *Manager) Disable(name string) error { // Enable activates a plugin, which implies that they are ready to be used by containers. func (pm *Manager) Enable(name string, config *types.PluginEnableConfig) error { - p, err := pm.pluginStore.GetByName(name) if err != nil { return err } - p.TimeoutInSecs = config.Timeout - if err := pm.enable(p, false); err != nil { + c := &controller{timeoutInSecs: config.Timeout} + if err := pm.enable(p, c, false); err != nil { return err } pm.pluginEventLogger(p.GetID(), name, "enable") @@ -267,25 +270,25 @@ func (pm *Manager) Push(name string, metaHeader http.Header, authConfig *types.A // Remove deletes plugin's root directory. func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error { p, err := pm.pluginStore.GetByName(name) + pm.mu.RLock() + c := pm.cMap[p] + pm.mu.RUnlock() + if err != nil { return err } if !config.ForceRemove { - p.RLock() - if p.RefCount > 0 { - p.RUnlock() + if p.GetRefCount() > 0 { return fmt.Errorf("plugin %s is in use", p.Name()) } - p.RUnlock() - if p.IsEnabled() { return fmt.Errorf("plugin %s is enabled", p.Name()) } } if p.IsEnabled() { - if err := pm.disable(p); err != nil { + if err := pm.disable(p, c); err != nil { logrus.Errorf("failed to disable plugin '%s': %s", p.Name(), err) } } diff --git a/plugin/manager.go b/plugin/manager.go index a8789833080b8448245574a84868fd5bba08bb55..0f6c43cdc853ebc996eb783ff3f4eedd4cf36f61 100644 --- a/plugin/manager.go +++ b/plugin/manager.go @@ -19,7 +19,7 @@ var ( ) func (pm *Manager) restorePlugin(p *v2.Plugin) error { - p.RuntimeSourcePath = filepath.Join(pm.runRoot, p.GetID()) + p.Restore(pm.runRoot) if p.IsEnabled() { return pm.restore(p) } @@ -37,6 +37,15 @@ type Manager struct { registryService registry.Service liveRestore bool pluginEventLogger eventLogger + mu sync.RWMutex // protects cMap + cMap map[*v2.Plugin]*controller +} + +// controller represents the manager's control on a plugin. +type controller struct { + restart bool + exitChan chan bool + timeoutInSecs int } // GetManager returns the singleton plugin Manager @@ -67,7 +76,8 @@ func Init(root string, ps *store.Store, remote libcontainerd.Remote, rs registry if err != nil { return err } - if err := manager.init(); err != nil { + manager.cMap = make(map[*v2.Plugin]*controller) + if err := manager.reload(); err != nil { return err } return nil @@ -83,22 +93,27 @@ func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error { if err != nil { return err } - p.RLock() - if p.ExitChan != nil { - close(p.ExitChan) + + pm.mu.RLock() + c := pm.cMap[p] + + if c.exitChan != nil { + close(c.exitChan) } - restart := p.Restart - p.RUnlock() + restart := c.restart + pm.mu.RUnlock() + p.RemoveFromDisk() if restart { - pm.enable(p, true) + pm.enable(p, c, true) } } return nil } -func (pm *Manager) init() error { +// reload is used on daemon restarts to load the manager's state +func (pm *Manager) reload() error { dt, err := os.Open(filepath.Join(pm.libRoot, "plugins.json")) if err != nil { if os.IsNotExist(err) { @@ -117,6 +132,8 @@ func (pm *Manager) init() error { var group sync.WaitGroup group.Add(len(plugins)) for _, p := range plugins { + c := &controller{} + pm.cMap[p] = c go func(p *v2.Plugin) { defer group.Done() if err := pm.restorePlugin(p); err != nil { @@ -129,7 +146,7 @@ func (pm *Manager) init() error { if requiresManualRestore { // if liveRestore is not enabled, the plugin will be stopped now so we should enable it - if err := pm.enable(p, true); err != nil { + if err := pm.enable(p, c, true); err != nil { logrus.Errorf("failed to enable plugin '%s': %s", p.Name(), err) } } diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go index bf477f4163b7c96c02c53afe540be1847690f385..b1bf221fda978bd935367b82be35e12df1158511 100644 --- a/plugin/manager_linux.go +++ b/plugin/manager_linux.go @@ -16,7 +16,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -func (pm *Manager) enable(p *v2.Plugin, force bool) error { +func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { if p.IsEnabled() && !force { return fmt.Errorf("plugin %s is already enabled", p.Name()) } @@ -24,23 +24,26 @@ func (pm *Manager) enable(p *v2.Plugin, force bool) error { if err != nil { return err } - p.Lock() - p.Restart = true - p.ExitChan = make(chan bool) - p.Unlock() + + c.restart = true + c.exitChan = make(chan bool) + + pm.mu.Lock() + pm.cMap[p] = c + pm.mu.Unlock() + if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil { return err } - p.PClient, err = plugins.NewClientWithTimeout("unix://"+filepath.Join(p.RuntimeSourcePath, p.GetSocket()), nil, p.TimeoutInSecs) + client, err := plugins.NewClientWithTimeout("unix://"+filepath.Join(p.GetRuntimeSourcePath(), p.GetSocket()), nil, c.timeoutInSecs) if err != nil { - p.Lock() - p.Restart = false - p.Unlock() - shutdownPlugin(p, pm.containerdClient) + c.restart = false + shutdownPlugin(p, c, pm.containerdClient) return err } + p.SetPClient(client) pm.pluginStore.SetState(p, true) pm.pluginStore.CallHandler(p) @@ -51,7 +54,7 @@ func (pm *Manager) restore(p *v2.Plugin) error { return pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())) } -func shutdownPlugin(p *v2.Plugin, containerdClient libcontainerd.Client) { +func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) { pluginID := p.GetID() err := containerdClient.Signal(pluginID, int(syscall.SIGTERM)) @@ -59,7 +62,7 @@ func shutdownPlugin(p *v2.Plugin, containerdClient libcontainerd.Client) { logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err) } else { select { - case <-p.ExitChan: + case <-c.exitChan: logrus.Debug("Clean shutdown of plugin") case <-time.After(time.Second * 10): logrus.Debug("Force shutdown plugin") @@ -70,15 +73,13 @@ func shutdownPlugin(p *v2.Plugin, containerdClient libcontainerd.Client) { } } -func (pm *Manager) disable(p *v2.Plugin) error { +func (pm *Manager) disable(p *v2.Plugin, c *controller) error { if !p.IsEnabled() { return fmt.Errorf("plugin %s is already disabled", p.Name()) } - p.Lock() - p.Restart = false - p.Unlock() - shutdownPlugin(p, pm.containerdClient) + c.restart = false + shutdownPlugin(p, c, pm.containerdClient) pm.pluginStore.SetState(p, false) return nil } @@ -87,15 +88,17 @@ func (pm *Manager) disable(p *v2.Plugin) error { func (pm *Manager) Shutdown() { plugins := pm.pluginStore.GetAll() for _, p := range plugins { + pm.mu.RLock() + c := pm.cMap[p] + pm.mu.RUnlock() + if pm.liveRestore && p.IsEnabled() { logrus.Debug("Plugin active when liveRestore is set, skipping shutdown") continue } if pm.containerdClient != nil && p.IsEnabled() { - p.Lock() - p.Restart = false - p.Unlock() - shutdownPlugin(p, pm.containerdClient) + c.restart = false + shutdownPlugin(p, c, pm.containerdClient) } } } diff --git a/plugin/manager_solaris.go b/plugin/manager_solaris.go index 7656a59ad76f98e551b39cd984d76d855834b26b..72ccae72d3208487cb35a2800b3741c1773c3da8 100644 --- a/plugin/manager_solaris.go +++ b/plugin/manager_solaris.go @@ -7,7 +7,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -func (pm *Manager) enable(p *v2.Plugin, force bool) error { +func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { return fmt.Errorf("Not implemented") } @@ -15,7 +15,7 @@ func (pm *Manager) initSpec(p *v2.Plugin) (*specs.Spec, error) { return nil, fmt.Errorf("Not implemented") } -func (pm *Manager) disable(p *v2.Plugin) error { +func (pm *Manager) disable(p *v2.Plugin, c *controller) error { return fmt.Errorf("Not implemented") } diff --git a/plugin/manager_windows.go b/plugin/manager_windows.go index 682801875057d1e6d8177d4e8f6d02e461897ba6..4469a671f7dd7a53f4d9ac8104fdb3a35b0b6a16 100644 --- a/plugin/manager_windows.go +++ b/plugin/manager_windows.go @@ -9,7 +9,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -func (pm *Manager) enable(p *v2.Plugin, force bool) error { +func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { return fmt.Errorf("Not implemented") } @@ -17,7 +17,7 @@ func (pm *Manager) initSpec(p *v2.Plugin) (*specs.Spec, error) { return nil, fmt.Errorf("Not implemented") } -func (pm *Manager) disable(p *v2.Plugin) error { +func (pm *Manager) disable(p *v2.Plugin, c *controller) error { return fmt.Errorf("Not implemented") } diff --git a/plugin/store/store.go b/plugin/store/store.go index 0517f8f2c744726cacf4e0ad361b9727e6baaa63..4f4665eb42bed2c540159d03e63910586be748e8 100644 --- a/plugin/store/store.go +++ b/plugin/store/store.go @@ -174,9 +174,7 @@ func (ps *Store) Get(name, capability string, mode int) (plugingetter.CompatPlug } p, err = ps.GetByName(fullName) if err == nil { - p.Lock() - p.RefCount += mode - p.Unlock() + p.SetRefCount(mode + p.GetRefCount()) if p.IsEnabled() { return p.FilterByCap(capability) } diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go index 7ea115cb3978d94f59e0ab53748b1fda2506fef2..4046bf7dbeb43d774e616de10d1a83bbc1870e45 100644 --- a/plugin/v2/plugin.go +++ b/plugin/v2/plugin.go @@ -17,15 +17,12 @@ import ( // Plugin represents an individual plugin. type Plugin struct { - sync.RWMutex - PluginObj types.Plugin `json:"plugin"` - PClient *plugins.Client `json:"-"` - RuntimeSourcePath string `json:"-"` - RefCount int `json:"-"` - Restart bool `json:"-"` - ExitChan chan bool `json:"-"` - LibRoot string `json:"-"` - TimeoutInSecs int `json:"-"` + mu sync.RWMutex + PluginObj types.Plugin `json:"plugin"` + pClient *plugins.Client + runtimeSourcePath string + refCount int + libRoot string } const defaultPluginRuntimeDestination = "/run/docker/plugins" @@ -47,14 +44,39 @@ func newPluginObj(name, id, tag string) types.Plugin { func NewPlugin(name, id, runRoot, libRoot, tag string) *Plugin { return &Plugin{ PluginObj: newPluginObj(name, id, tag), - RuntimeSourcePath: filepath.Join(runRoot, id), - LibRoot: libRoot, + runtimeSourcePath: filepath.Join(runRoot, id), + libRoot: libRoot, } } +// Restore restores the plugin +func (p *Plugin) Restore(runRoot string) { + p.runtimeSourcePath = filepath.Join(runRoot, p.GetID()) +} + +// GetRuntimeSourcePath gets the Source (host) path of the plugin socket +// This path gets bind mounted into the plugin. +func (p *Plugin) GetRuntimeSourcePath() string { + p.mu.RLock() + defer p.mu.RUnlock() + + return p.runtimeSourcePath +} + // Client returns the plugin client. func (p *Plugin) Client() *plugins.Client { - return p.PClient + p.mu.RLock() + defer p.mu.RUnlock() + + return p.pClient +} + +// SetPClient set the plugin client. +func (p *Plugin) SetPClient(client *plugins.Client) { + p.mu.Lock() + defer p.mu.Unlock() + + p.pClient = client } // IsV1 returns true for V1 plugins and false otherwise. @@ -85,12 +107,12 @@ func (p *Plugin) FilterByCap(capability string) (*Plugin, error) { // RemoveFromDisk deletes the plugin's runtime files from disk. func (p *Plugin) RemoveFromDisk() error { - return os.RemoveAll(p.RuntimeSourcePath) + return os.RemoveAll(p.runtimeSourcePath) } // InitPlugin populates the plugin object from the plugin config file. func (p *Plugin) InitPlugin() error { - dt, err := os.Open(filepath.Join(p.LibRoot, p.PluginObj.ID, "config.json")) + dt, err := os.Open(filepath.Join(p.libRoot, p.PluginObj.ID, "config.json")) if err != nil { return err } @@ -118,7 +140,7 @@ func (p *Plugin) InitPlugin() error { } func (p *Plugin) writeSettings() error { - f, err := os.Create(filepath.Join(p.LibRoot, p.PluginObj.ID, "plugin-settings.json")) + f, err := os.Create(filepath.Join(p.libRoot, p.PluginObj.ID, "plugin-settings.json")) if err != nil { return err } @@ -129,8 +151,8 @@ func (p *Plugin) writeSettings() error { // Set is used to pass arguments to the plugin. func (p *Plugin) Set(args []string) error { - p.Lock() - defer p.Unlock() + p.mu.Lock() + defer p.mu.Unlock() if p.PluginObj.Enabled { return fmt.Errorf("cannot set on an active plugin, disable plugin before setting") @@ -218,36 +240,52 @@ next: // IsEnabled returns the active state of the plugin. func (p *Plugin) IsEnabled() bool { - p.RLock() - defer p.RUnlock() + p.mu.RLock() + defer p.mu.RUnlock() return p.PluginObj.Enabled } // GetID returns the plugin's ID. func (p *Plugin) GetID() string { - p.RLock() - defer p.RUnlock() + p.mu.RLock() + defer p.mu.RUnlock() return p.PluginObj.ID } // GetSocket returns the plugin socket. func (p *Plugin) GetSocket() string { - p.RLock() - defer p.RUnlock() + p.mu.RLock() + defer p.mu.RUnlock() return p.PluginObj.Config.Interface.Socket } // GetTypes returns the interface types of a plugin. func (p *Plugin) GetTypes() []types.PluginInterfaceType { - p.RLock() - defer p.RUnlock() + p.mu.RLock() + defer p.mu.RUnlock() return p.PluginObj.Config.Interface.Types } +// GetRefCount returns the reference count. +func (p *Plugin) GetRefCount() int { + p.mu.RLock() + defer p.mu.RUnlock() + + return p.refCount +} + +// SetRefCount sets the reference count. +func (p *Plugin) SetRefCount(count int) { + p.mu.Lock() + defer p.mu.Unlock() + + p.refCount = count +} + // InitSpec creates an OCI spec from the plugin's config. func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) { rootfs := filepath.Join(libRoot, p.PluginObj.ID, "rootfs") @@ -262,7 +300,7 @@ func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) { } mounts := append(p.PluginObj.Config.Mounts, types.PluginMount{ - Source: &p.RuntimeSourcePath, + Source: &p.runtimeSourcePath, Destination: defaultPluginRuntimeDestination, Type: "bind", Options: []string{"rbind", "rshared"}, From d842cf6db627fd3ffdd0615b8089412f8beda951 Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Thu, 8 Dec 2016 16:57:27 +0800 Subject: [PATCH 29/68] add SCOPE field content for docker network ls Signed-off-by: lixiaobing10051267 (cherry picked from commit 170fcead7ecd0dbe323d727a2afd3e2694e6af29) Signed-off-by: Victor Vieux --- docs/reference/commandline/network_ls.md | 2 +- man/docker-network-ls.1.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/commandline/network_ls.md b/docs/reference/commandline/network_ls.md index 42bde111c4bd78026a7b051216e98ae625165bbd..a4f671d569e2fa702d498dddb5a1d3512124f255 100644 --- a/docs/reference/commandline/network_ls.md +++ b/docs/reference/commandline/network_ls.md @@ -40,7 +40,7 @@ NETWORK ID NAME DRIVER SCOPE 7fca4eb8c647 bridge bridge local 9f904ee27bf5 none null local cf03ee007fb4 host host local -78b03ee04fc4 multi-host overlay local +78b03ee04fc4 multi-host overlay swarm ``` Use the `--no-trunc` option to display the full network id: diff --git a/man/docker-network-ls.1.md b/man/docker-network-ls.1.md index 4166ba967ae87b4502593567afeae496183a33f1..f319e66035a0c559d7801bfa88d2d3ad817314e7 100644 --- a/man/docker-network-ls.1.md +++ b/man/docker-network-ls.1.md @@ -19,11 +19,11 @@ networks that span across multiple hosts in a cluster, for example: ```bash $ docker network ls - NETWORK ID NAME DRIVER - 7fca4eb8c647 bridge bridge - 9f904ee27bf5 none null - cf03ee007fb4 host host - 78b03ee04fc4 multi-host overlay + NETWORK ID NAME DRIVER SCOPE + 7fca4eb8c647 bridge bridge local + 9f904ee27bf5 none null local + cf03ee007fb4 host host local + 78b03ee04fc4 multi-host overlay swarm ``` Use the `--no-trunc` option to display the full network id: From aa22b964577f7ae27ce943045975fb83d049cf94 Mon Sep 17 00:00:00 2001 From: Qinglan Peng Date: Thu, 8 Dec 2016 19:03:46 +0800 Subject: [PATCH 30/68] fix some version information Signed-off-by: Qinglan Peng (cherry picked from commit aa26364ce1de66b23e51864682cbdadb4a272fa4) Signed-off-by: Victor Vieux --- docs/deprecated.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/deprecated.md b/docs/deprecated.md index d6cfef5d033a81e1902bd69f7b952294bee0de15..b8e6af1116411de55a3ba04b1b22228ec5417c9e 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -21,28 +21,28 @@ To learn more about Docker Engine's deprecation policy, see [Feature Deprecation Policy](https://docs.docker.com/engine/#feature-deprecation-policy). ## `filter` param for `/images/json` endpoint -**Deprecated In Release: [v1.13](https://github.com/docker/docker/releases/tag/v1.13.0)** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** The `filter` param to filter the list of image by reference (name or name:tag) is now implemented as a regular filter, named `reference`. ### `repository:shortid` image references -**Deprecated In Release: [v1.13](https://github.com/docker/docker/releases/tag/v1.13.0)** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** `repository:shortid` syntax for referencing images is very little used, collides with tag references can be confused with digest references. ### `docker daemon` subcommand -**Deprecated In Release: [v1.13](https://github.com/docker/docker/releases/tag/v1.13.0)** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** The daemon is moved to a separate binary (`dockerd`), and should be used instead. ### Duplicate keys with conflicting values in engine labels -**Deprecated In Release: [v1.13](https://github.com/docker/docker/releases/tag/v1.13.0)** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** @@ -50,12 +50,12 @@ Duplicate keys with conflicting values have been deprecated. A warning is displa in the output, and an error will be returned in the future. ### `MAINTAINER` in Dockerfile -**Deprecated In Release: v1.13.0** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** `MAINTAINER` was an early very limited form of `LABEL` which should be used instead. ### API calls without a version -**Deprecated In Release: [v1.13](https://github.com/docker/docker/releases/)** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** @@ -64,7 +64,7 @@ future Engine versions. Instead of just requesting, for example, the URL `/containers/json`, you must now request `/v1.25/containers/json`. ### Backing filesystem without `d_type` support for overlay/overlay2 -**Deprecated In Release: v1.13.0** +**Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** **Target For Removal In Release: v1.16** @@ -75,12 +75,12 @@ if it is formatted with the `ftype=0` option. Please also refer to [#27358](https://github.com/docker/docker/issues/27358) for further information. -### Three argument form in `docker import` +### Three arguments form in `docker import` **Deprecated In Release: [v0.6.7](https://github.com/docker/docker/releases/tag/v0.6.7)** **Removed In Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)** -The `docker import` command format 'file|URL|- [REPOSITORY [TAG]]' is deprecated since November 2013. It's no more supported. +The `docker import` command format `file|URL|- [REPOSITORY [TAG]]` is deprecated since November 2013. It's no more supported. ### `-h` shorthand for `--help` @@ -136,7 +136,7 @@ To make tagging consistent across the various `docker` commands, the `-f` flag o Passing an `HostConfig` to `POST /containers/{name}/start` is deprecated in favor of defining it at container creation (`POST /containers/create`). -### Docker ps 'before' and 'since' options +### `--before` and `--since` flags on `docker ps` **Deprecated In Release: [v1.10.0](https://github.com/docker/docker/releases/tag/v1.10.0)** @@ -145,7 +145,7 @@ defining it at container creation (`POST /containers/create`). The `docker ps --before` and `docker ps --since` options are deprecated. Use `docker ps --filter=before=...` and `docker ps --filter=since=...` instead. -### Docker search 'automated' and 'stars' options +### `--automated` and `--stars` flags on `docker search` **Deprecated in Release: [v1.12.0](https://github.com/docker/docker/releases/tag/v1.12.0)** @@ -234,7 +234,7 @@ The single-dash (`-help`) was removed, in favor of the double-dash `--help` **Deprecated In Release: [v0.10.0](https://github.com/docker/docker/releases/tag/v0.10.0)** -**Removed In Release: [v1.13.0](https://github.com/docker/docker/releases/)** +**Removed In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** The flag `--run` of the docker commit (and its short version `-run`) were deprecated in favor of the `--changes` flag that allows to pass `Dockerfile` commands. From 3a571b72fd9b2d25190464a954952f8d806acab1 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 22 Nov 2016 11:21:34 -0800 Subject: [PATCH 31/68] plugins: container-rootfs-relative paths Legacy plugins expect host-relative paths (such as for Volume.Mount). However, a containerized plugin cannot respond with a host-relative path. Therefore, this commit modifies new volume plugins' paths in Mount and List to prepend the container's rootfs path. This introduces a new PropagatedMount field in the Plugin Config. When it is set for volume plugins, RootfsPropagation is set to rshared and the path specified by PropagatedMount is bind-mounted with rshared prior to launching the container. This is so that the daemon code can access the paths returned by the plugin from the host mount namespace. Signed-off-by: Tibor Vass (cherry picked from commit c54b717caf1a55e525ce180bfcb42addd59c6633) Signed-off-by: Victor Vieux --- api/swagger.yaml | 4 ++ api/types/plugin.go | 4 ++ docs/extend/config.md | 4 ++ docs/extend/plugins_volume.md | 4 ++ integration-cli/daemon.go | 5 ++ .../docker_cli_daemon_plugins_test.go | 35 ++++++++-- integration-cli/docker_cli_plugins_test.go | 5 +- pkg/plugingetter/getter.go | 1 + pkg/plugins/plugins.go | 6 ++ plugin/backend_linux.go | 10 ++- plugin/distribution/pull.go | 3 +- plugin/manager.go | 29 +++++++- plugin/manager_linux.go | 10 ++- plugin/v2/plugin.go | 67 ++++++++++--------- volume/drivers/adapter.go | 54 +++++++++------ volume/drivers/extpoint.go | 8 +-- volume/drivers/proxy.go | 1 + 17 files changed, 182 insertions(+), 68 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 13699db2c89037051aa5379d0588f431cd342f6b..ffbeffb2ce91f25c2d207f20794cc00258e880ed 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1382,6 +1382,7 @@ definitions: - Workdir - Network - Linux + - PropagatedMount - Mounts - Env - Args @@ -1446,6 +1447,9 @@ definitions: type: "array" items: $ref: "#/definitions/PluginDevice" + PropagatedMount: + type: "string" + x-nullable: false Mounts: type: "array" items: diff --git a/api/types/plugin.go b/api/types/plugin.go index 2bd2a6eeacbe5c83050f73712461ce8165d15743..1f46408b927ad76f8422f10226b5f1f7c5eb66e9 100644 --- a/api/types/plugin.go +++ b/api/types/plugin.go @@ -71,6 +71,10 @@ type PluginConfig struct { // Required: true Network PluginConfigNetwork `json:"Network"` + // propagated mount + // Required: true + PropagatedMount string `json:"PropagatedMount"` + // user User PluginConfigUser `json:"User,omitempty"` diff --git a/docs/extend/config.md b/docs/extend/config.md index 221150a07b6fd2ff9664e74ee11e4f5397021077..e068eaccfaf141d4ba38b0ec2d063258452337c0 100644 --- a/docs/extend/config.md +++ b/docs/extend/config.md @@ -111,6 +111,10 @@ Config provides the base accessible fields for working with V0 plugin format options of the mount. +- **`propagatedMount`** *string* + + path to be mounted as rshared, so that mounts under that path are visible to docker. This is useful for volume plugins. + - **`env`** *PluginEnv array* env of the plugin, struct consisting of the following fields diff --git a/docs/extend/plugins_volume.md b/docs/extend/plugins_volume.md index b123543f6f4ad78f847d78b8c9c90ff29f58e27e..c060bf39b1851bd523158c0990ef0e46b220782f 100644 --- a/docs/extend/plugins_volume.md +++ b/docs/extend/plugins_volume.md @@ -22,6 +22,10 @@ beyond the lifetime of a single Engine host. See the ## Changelog +### 1.13.0 + +- If used as part of the v2 plugin architecture, mountpoints that are part of paths returned by plugin have to be mounted under the directory specified by PropagatedMount in the plugin configuration [#26398](https://github.com/docker/docker/pull/26398) + ### 1.12.0 - Add `Status` field to `VolumeDriver.Get` response ([#21006](https://github.com/docker/docker/pull/21006#)) diff --git a/integration-cli/daemon.go b/integration-cli/daemon.go index 97819e1ab17593c57f8029ecf7cae9d92aa76406..9fd3f1e82d858277d61cefff4ece1d27037e0f0b 100644 --- a/integration-cli/daemon.go +++ b/integration-cli/daemon.go @@ -88,6 +88,11 @@ func NewDaemon(c *check.C) *Daemon { } } +// RootDir returns the root directory of the daemon. +func (d *Daemon) RootDir() string { + return d.root +} + func (d *Daemon) getClientConfig() (*clientConfig, error) { var ( transport *http.Transport diff --git a/integration-cli/docker_cli_daemon_plugins_test.go b/integration-cli/docker_cli_daemon_plugins_test.go index 8edc3e66fcb0e9c7a5e012f16f96545504bee8e8..7ee068b37abac48e939cc4ff401cfd18f8f759a0 100644 --- a/integration-cli/docker_cli_daemon_plugins_test.go +++ b/integration-cli/docker_cli_daemon_plugins_test.go @@ -10,6 +10,7 @@ import ( "syscall" "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/docker/pkg/mount" "github.com/go-check/check" ) @@ -186,7 +187,6 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) { testRequires(c, Network, IsAmd64) volName := "plugin-volume" - volRoot := "/data" destDir := "/tmp/data/" destFile := "foo" @@ -197,13 +197,25 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) { if err != nil { c.Fatalf("Could not install plugin: %v %s", err, out) } + pluginID, err := s.d.Cmd("plugin", "inspect", "-f", "{{.Id}}", pName) + pluginID = strings.TrimSpace(pluginID) + if err != nil { + c.Fatalf("Could not retrieve plugin ID: %v %s", err, pluginID) + } + mountpointPrefix := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs") defer func() { if out, err := s.d.Cmd("plugin", "disable", pName); err != nil { c.Fatalf("Could not disable plugin: %v %s", err, out) } + if out, err := s.d.Cmd("plugin", "remove", pName); err != nil { c.Fatalf("Could not remove plugin: %v %s", err, out) } + + exists, err := existsMountpointWithPrefix(mountpointPrefix) + c.Assert(err, checker.IsNil) + c.Assert(exists, checker.Equals, false) + }() out, err = s.d.Cmd("volume", "create", "-d", pName, volName) @@ -231,11 +243,24 @@ func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) { out, err = s.d.Cmd("run", "--rm", "-v", volName+":"+destDir, "busybox", "touch", destDir+destFile) c.Assert(err, checker.IsNil, check.Commentf(out)) - path := filepath.Join(mountPoint, destFile) + path := filepath.Join(s.d.RootDir(), "plugins", pluginID, "rootfs", mountPoint, destFile) _, err = os.Lstat(path) c.Assert(err, checker.IsNil) - // tiborvass/no-remove is a volume plugin that persists data on disk at /data, - // even after the volume is removed. So perform an explicit filesystem cleanup. - os.RemoveAll(volRoot) + exists, err := existsMountpointWithPrefix(mountpointPrefix) + c.Assert(err, checker.IsNil) + c.Assert(exists, checker.Equals, true) +} + +func existsMountpointWithPrefix(mountpointPrefix string) (bool, error) { + mounts, err := mount.GetMounts() + if err != nil { + return false, err + } + for _, mnt := range mounts { + if strings.HasPrefix(mnt.Mountpoint, mountpointPrefix) { + return true, nil + } + } + return false, nil } diff --git a/integration-cli/docker_cli_plugins_test.go b/integration-cli/docker_cli_plugins_test.go index 730580e568b34c112359d3edc867fbdb96744391..7b9a69cb7504f10f9a548125f92604a42814ebcc 100644 --- a/integration-cli/docker_cli_plugins_test.go +++ b/integration-cli/docker_cli_plugins_test.go @@ -11,8 +11,8 @@ import ( ) var ( - pluginProcessName = "no-remove" - pName = "tiborvass/no-remove" + pluginProcessName = "sample-volume-plugin" + pName = "tiborvass/sample-volume-plugin" pTag = "latest" pNameWithTag = pName + ":" + pTag ) @@ -33,6 +33,7 @@ func (s *DockerSuite) TestPluginBasicOps(c *check.C) { c.Assert(err, checker.IsNil) out, _, err = dockerCmdWithError("plugin", "remove", pNameWithTag) + c.Assert(err, checker.NotNil) c.Assert(out, checker.Contains, "is enabled") _, _, err = dockerCmdWithError("plugin", "disable", pNameWithTag) diff --git a/pkg/plugingetter/getter.go b/pkg/plugingetter/getter.go index cd8214cae988ab9841b0bf975224f7af1a8a964d..c568184b3ce47494b384b0d3b64a39e36f6787ca 100644 --- a/pkg/plugingetter/getter.go +++ b/pkg/plugingetter/getter.go @@ -15,6 +15,7 @@ const ( type CompatPlugin interface { Client() *plugins.Client Name() string + BasePath() string IsV1() bool } diff --git a/pkg/plugins/plugins.go b/pkg/plugins/plugins.go index 432c47d8e57dbdf4b1f220103e23912373e9e3ee..acfb20999256b19235711726a69aa32afbaeaca1 100644 --- a/pkg/plugins/plugins.go +++ b/pkg/plugins/plugins.go @@ -78,6 +78,12 @@ type Plugin struct { activateWait *sync.Cond } +// BasePath returns the path to which all paths returned by the plugin are relative to. +// For v1 plugins, this always returns the host's root directory. +func (p *Plugin) BasePath() string { + return "/" +} + // Name returns the name of the plugin. func (p *Plugin) Name() string { return p.name diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go index f9396626c3201f3094719f9a1f953dfa0810ea8e..416edc34774fb7b49f3cab02bef9d92c0a7af9d6 100644 --- a/plugin/backend_linux.go +++ b/plugin/backend_linux.go @@ -255,7 +255,7 @@ func (pm *Manager) Push(name string, metaHeader http.Header, authConfig *types.A return err } - rootfs, err := archive.Tar(filepath.Join(dest, "rootfs"), archive.Gzip) + rootfs, err := archive.Tar(p.Rootfs, archive.Gzip) if err != nil { return err } @@ -293,9 +293,13 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error { } } + id := p.GetID() pm.pluginStore.Remove(p) - os.RemoveAll(filepath.Join(pm.libRoot, p.GetID())) - pm.pluginEventLogger(p.GetID(), name, "remove") + pluginDir := filepath.Join(pm.libRoot, id) + if err := os.RemoveAll(pluginDir); err != nil { + logrus.Warnf("unable to remove %q from plugin remove: %v", pluginDir, err) + } + pm.pluginEventLogger(id, name, "remove") return nil } diff --git a/plugin/distribution/pull.go b/plugin/distribution/pull.go index 4e8992cc2ea5e3c5fb5f0662c67cd9b782143144..3e185cb0d31ef17f2f9712077faacabad6eefe7b 100644 --- a/plugin/distribution/pull.go +++ b/plugin/distribution/pull.go @@ -177,7 +177,8 @@ func WritePullData(pd PullData, dest string, extract bool) error { if err := json.Unmarshal(config, &p); err != nil { return err } - logrus.Debugf("%#v", p) + logrus.Debugf("plugin: %#v", p) + if err := os.MkdirAll(dest, 0700); err != nil { return err } diff --git a/plugin/manager.go b/plugin/manager.go index 0f6c43cdc853ebc996eb783ff3f4eedd4cf36f61..bf20990c0855b6b92e0dd94bb184616eb8c7cf42 100644 --- a/plugin/manager.go +++ b/plugin/manager.go @@ -5,10 +5,12 @@ import ( "io" "os" "path/filepath" + "strings" "sync" "github.com/Sirupsen/logrus" "github.com/docker/docker/libcontainerd" + "github.com/docker/docker/pkg/mount" "github.com/docker/docker/plugin/store" "github.com/docker/docker/plugin/v2" "github.com/docker/docker/registry" @@ -63,7 +65,7 @@ func Init(root string, ps *store.Store, remote libcontainerd.Remote, rs registry root = filepath.Join(root, "plugins") manager = &Manager{ libRoot: root, - runRoot: "/run/docker", + runRoot: "/run/docker/plugins", pluginStore: ps, registryService: rs, liveRestore: liveRestore, @@ -104,6 +106,13 @@ func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error { pm.mu.RUnlock() p.RemoveFromDisk() + + if p.PropagatedMount != "" { + if err := mount.Unmount(p.PropagatedMount); err != nil { + logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err) + } + } + if restart { pm.enable(p, c, true) } @@ -141,6 +150,24 @@ func (pm *Manager) reload() error { return } + if p.Rootfs != "" { + p.Rootfs = filepath.Join(pm.libRoot, p.PluginObj.ID, "rootfs") + } + + // We should only enable rootfs propagation for certain plugin types that need it. + for _, typ := range p.PluginObj.Config.Interface.Types { + if typ.Capability == "volumedriver" && typ.Prefix == "docker" && strings.HasPrefix(typ.Version, "1.") { + if p.PluginObj.Config.PropagatedMount != "" { + // TODO: sanitize PropagatedMount and prevent breakout + p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount) + if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil { + logrus.Errorf("failed to create PropagatedMount directory at %s: %v", p.PropagatedMount, err) + return + } + } + } + } + pm.pluginStore.Update(p) requiresManualRestore := !pm.liveRestore && p.IsEnabled() diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go index b1bf221fda978bd935367b82be35e12df1158511..9764ec6a74aef5bbd387dfbc90394b2dc2cfb9c1 100644 --- a/plugin/manager_linux.go +++ b/plugin/manager_linux.go @@ -11,16 +11,18 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/libcontainerd" "github.com/docker/docker/oci" + "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/plugins" "github.com/docker/docker/plugin/v2" specs "github.com/opencontainers/runtime-spec/specs-go" ) func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { + p.Rootfs = filepath.Join(pm.libRoot, p.PluginObj.ID, "rootfs") if p.IsEnabled() && !force { return fmt.Errorf("plugin %s is already enabled", p.Name()) } - spec, err := p.InitSpec(oci.DefaultSpec(), pm.libRoot) + spec, err := p.InitSpec(oci.DefaultSpec()) if err != nil { return err } @@ -32,6 +34,12 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { pm.cMap[p] = c pm.mu.Unlock() + if p.PropagatedMount != "" { + if err := mount.MakeRShared(p.PropagatedMount); err != nil { + return err + } + } + if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil { return err } diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go index 4046bf7dbeb43d774e616de10d1a83bbc1870e45..be4a87d05fc87ce250a877093d6b500180f22491 100644 --- a/plugin/v2/plugin.go +++ b/plugin/v2/plugin.go @@ -2,6 +2,7 @@ package v2 import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -22,7 +23,9 @@ type Plugin struct { pClient *plugins.Client runtimeSourcePath string refCount int - libRoot string + LibRoot string // TODO: make private + PropagatedMount string // TODO: make private + Rootfs string // TODO: make private } const defaultPluginRuntimeDestination = "/run/docker/plugins" @@ -45,7 +48,7 @@ func NewPlugin(name, id, runRoot, libRoot, tag string) *Plugin { return &Plugin{ PluginObj: newPluginObj(name, id, tag), runtimeSourcePath: filepath.Join(runRoot, id), - libRoot: libRoot, + LibRoot: libRoot, } } @@ -63,6 +66,12 @@ func (p *Plugin) GetRuntimeSourcePath() string { return p.runtimeSourcePath } +// BasePath returns the path to which all paths returned by the plugin are relative to. +// For Plugin objects this returns the host path of the plugin container's rootfs. +func (p *Plugin) BasePath() string { + return p.Rootfs +} + // Client returns the plugin client. func (p *Plugin) Client() *plugins.Client { p.mu.RLock() @@ -112,7 +121,7 @@ func (p *Plugin) RemoveFromDisk() error { // InitPlugin populates the plugin object from the plugin config file. func (p *Plugin) InitPlugin() error { - dt, err := os.Open(filepath.Join(p.libRoot, p.PluginObj.ID, "config.json")) + dt, err := os.Open(filepath.Join(p.LibRoot, p.PluginObj.ID, "config.json")) if err != nil { return err } @@ -123,9 +132,7 @@ func (p *Plugin) InitPlugin() error { } p.PluginObj.Settings.Mounts = make([]types.PluginMount, len(p.PluginObj.Config.Mounts)) - for i, mount := range p.PluginObj.Config.Mounts { - p.PluginObj.Settings.Mounts[i] = mount - } + copy(p.PluginObj.Settings.Mounts, p.PluginObj.Config.Mounts) p.PluginObj.Settings.Env = make([]string, 0, len(p.PluginObj.Config.Env)) p.PluginObj.Settings.Devices = make([]types.PluginDevice, 0, len(p.PluginObj.Config.Linux.Devices)) copy(p.PluginObj.Settings.Devices, p.PluginObj.Config.Linux.Devices) @@ -134,13 +141,14 @@ func (p *Plugin) InitPlugin() error { p.PluginObj.Settings.Env = append(p.PluginObj.Settings.Env, fmt.Sprintf("%s=%s", env.Name, *env.Value)) } } + p.PluginObj.Settings.Args = make([]string, len(p.PluginObj.Config.Args.Value)) copy(p.PluginObj.Settings.Args, p.PluginObj.Config.Args.Value) return p.writeSettings() } func (p *Plugin) writeSettings() error { - f, err := os.Create(filepath.Join(p.libRoot, p.PluginObj.ID, "plugin-settings.json")) + f, err := os.Create(filepath.Join(p.LibRoot, p.PluginObj.ID, "plugin-settings.json")) if err != nil { return err } @@ -287,18 +295,21 @@ func (p *Plugin) SetRefCount(count int) { } // InitSpec creates an OCI spec from the plugin's config. -func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) { - rootfs := filepath.Join(libRoot, p.PluginObj.ID, "rootfs") +func (p *Plugin) InitSpec(s specs.Spec) (*specs.Spec, error) { s.Root = specs.Root{ - Path: rootfs, + Path: p.Rootfs, Readonly: false, // TODO: all plugins should be readonly? settable in config? } - userMounts := make(map[string]struct{}, len(p.PluginObj.Config.Mounts)) - for _, m := range p.PluginObj.Config.Mounts { + userMounts := make(map[string]struct{}, len(p.PluginObj.Settings.Mounts)) + for _, m := range p.PluginObj.Settings.Mounts { userMounts[m.Destination] = struct{}{} } + if err := os.MkdirAll(p.runtimeSourcePath, 0755); err != nil { + return nil, err + } + mounts := append(p.PluginObj.Config.Mounts, types.PluginMount{ Source: &p.runtimeSourcePath, Destination: defaultPluginRuntimeDestination, @@ -328,27 +339,16 @@ func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) { }) } - for _, mount := range mounts { + for _, mnt := range mounts { m := specs.Mount{ - Destination: mount.Destination, - Type: mount.Type, - Options: mount.Options, + Destination: mnt.Destination, + Type: mnt.Type, + Options: mnt.Options, } - // TODO: if nil, then it's required and user didn't set it - if mount.Source != nil { - m.Source = *mount.Source - } - if m.Source != "" && m.Type == "bind" { - fi, err := os.Lstat(filepath.Join(rootfs, m.Destination)) // TODO: followsymlinks - if err != nil { - return nil, err - } - if fi.IsDir() { - if err := os.MkdirAll(m.Source, 0700); err != nil { - return nil, err - } - } + if mnt.Source == nil { + return nil, errors.New("mount source is not specified") } + m.Source = *mnt.Source s.Mounts = append(s.Mounts, m) } @@ -360,11 +360,16 @@ func (p *Plugin) InitSpec(s specs.Spec, libRoot string) (*specs.Spec, error) { } } + if p.PluginObj.Config.PropagatedMount != "" { + p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount) + s.Linux.RootfsPropagation = "rshared" + } + if p.PluginObj.Config.Linux.DeviceCreation { rwm := "rwm" s.Linux.Resources.Devices = []specs.DeviceCgroup{{Allow: true, Access: &rwm}} } - for _, dev := range p.PluginObj.Config.Linux.Devices { + for _, dev := range p.PluginObj.Settings.Devices { path := *dev.Path d, dPermissions, err := oci.DevicesFromPath(path, path, "rwm") if err != nil { diff --git a/volume/drivers/adapter.go b/volume/drivers/adapter.go index 4ce84d4076c8fff77ce232382a2eb9b4ce377af5..62ef7dfe6013f97e53377f4336bed7ee55c72857 100644 --- a/volume/drivers/adapter.go +++ b/volume/drivers/adapter.go @@ -2,6 +2,7 @@ package volumedrivers import ( "errors" + "path/filepath" "strings" "github.com/Sirupsen/logrus" @@ -14,6 +15,7 @@ var ( type volumeDriverAdapter struct { name string + baseHostPath string capabilities *volume.Capability proxy *volumeDriverProxy } @@ -27,9 +29,10 @@ func (a *volumeDriverAdapter) Create(name string, opts map[string]string) (volum return nil, err } return &volumeAdapter{ - proxy: a.proxy, - name: name, - driverName: a.name, + proxy: a.proxy, + name: name, + driverName: a.name, + baseHostPath: a.baseHostPath, }, nil } @@ -37,6 +40,13 @@ func (a *volumeDriverAdapter) Remove(v volume.Volume) error { return a.proxy.Remove(v.Name()) } +func hostPath(baseHostPath, path string) string { + if baseHostPath != "" { + path = filepath.Join(baseHostPath, path) + } + return path +} + func (a *volumeDriverAdapter) List() ([]volume.Volume, error) { ls, err := a.proxy.List() if err != nil { @@ -46,10 +56,11 @@ func (a *volumeDriverAdapter) List() ([]volume.Volume, error) { var out []volume.Volume for _, vp := range ls { out = append(out, &volumeAdapter{ - proxy: a.proxy, - name: vp.Name, - driverName: a.name, - eMount: vp.Mountpoint, + proxy: a.proxy, + name: vp.Name, + baseHostPath: a.baseHostPath, + driverName: a.name, + eMount: hostPath(a.baseHostPath, vp.Mountpoint), }) } return out, nil @@ -67,11 +78,12 @@ func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) { } return &volumeAdapter{ - proxy: a.proxy, - name: v.Name, - driverName: a.Name(), - eMount: v.Mountpoint, - status: v.Status, + proxy: a.proxy, + name: v.Name, + driverName: a.Name(), + eMount: v.Mountpoint, + status: v.Status, + baseHostPath: a.baseHostPath, }, nil } @@ -108,11 +120,12 @@ func (a *volumeDriverAdapter) getCapabilities() volume.Capability { } type volumeAdapter struct { - proxy *volumeDriverProxy - name string - driverName string - eMount string // ephemeral host volume path - status map[string]interface{} + proxy *volumeDriverProxy + name string + baseHostPath string + driverName string + eMount string // ephemeral host volume path + status map[string]interface{} } type proxyVolume struct { @@ -131,7 +144,8 @@ func (a *volumeAdapter) DriverName() string { func (a *volumeAdapter) Path() string { if len(a.eMount) == 0 { - a.eMount, _ = a.proxy.Path(a.name) + mountpoint, _ := a.proxy.Path(a.name) + a.eMount = hostPath(a.baseHostPath, mountpoint) } return a.eMount } @@ -141,8 +155,8 @@ func (a *volumeAdapter) CachedPath() string { } func (a *volumeAdapter) Mount(id string) (string, error) { - var err error - a.eMount, err = a.proxy.Mount(a.name, id) + mountpoint, err := a.proxy.Mount(a.name, id) + a.eMount = hostPath(a.baseHostPath, mountpoint) return a.eMount, err } diff --git a/volume/drivers/extpoint.go b/volume/drivers/extpoint.go index a8c10746ec8a7cd78033408a9b5b6ba225d3c5b4..78f86948f6609daea27ad99309455f30541e9783 100644 --- a/volume/drivers/extpoint.go +++ b/volume/drivers/extpoint.go @@ -22,9 +22,9 @@ var drivers = &driverExtpoint{ const extName = "VolumeDriver" // NewVolumeDriver returns a driver has the given name mapped on the given client. -func NewVolumeDriver(name string, c client) volume.Driver { +func NewVolumeDriver(name string, baseHostPath string, c client) volume.Driver { proxy := &volumeDriverProxy{c} - return &volumeDriverAdapter{name: name, proxy: proxy} + return &volumeDriverAdapter{name: name, baseHostPath: baseHostPath, proxy: proxy} } // volumeDriver defines the available functions that volume plugins must implement. @@ -117,7 +117,7 @@ func lookup(name string, mode int) (volume.Driver, error) { return nil, fmt.Errorf("Error looking up volume plugin %s: %v", name, err) } - d := NewVolumeDriver(p.Name(), p.Client()) + d := NewVolumeDriver(p.Name(), p.BasePath(), p.Client()) if err := validateDriver(d); err != nil { return nil, err } @@ -199,7 +199,7 @@ func GetAllDrivers() ([]volume.Driver, error) { continue } - ext = NewVolumeDriver(name, p.Client()) + ext = NewVolumeDriver(name, p.BasePath(), p.Client()) if p.IsV1() { drivers.extensions[name] = ext } diff --git a/volume/drivers/proxy.go b/volume/drivers/proxy.go index 431cb2ec01cce67d9f1a1b63cf76fdbc993fd4c7..b23db6258f930503c7131d560aca9deab47d97bf 100644 --- a/volume/drivers/proxy.go +++ b/volume/drivers/proxy.go @@ -4,6 +4,7 @@ package volumedrivers import ( "errors" + "github.com/docker/docker/volume" ) From 1c858abc96112115c888cba34856e5aa569f8b40 Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Fri, 9 Dec 2016 09:34:30 -0800 Subject: [PATCH 32/68] Fix race in setting plugin refcounts. Signed-off-by: Anusha Ragunathan (cherry picked from commit 4c088d1e2ebfcc384a365734017988f4fd1c4605) Signed-off-by: Victor Vieux --- plugin/store/store.go | 2 +- plugin/v2/plugin.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/store/store.go b/plugin/store/store.go index 4f4665eb42bed2c540159d03e63910586be748e8..81e89581cc6ffeb41f89141541e514048dde38c0 100644 --- a/plugin/store/store.go +++ b/plugin/store/store.go @@ -174,7 +174,7 @@ func (ps *Store) Get(name, capability string, mode int) (plugingetter.CompatPlug } p, err = ps.GetByName(fullName) if err == nil { - p.SetRefCount(mode + p.GetRefCount()) + p.AddRefCount(mode) if p.IsEnabled() { return p.FilterByCap(capability) } diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go index be4a87d05fc87ce250a877093d6b500180f22491..ff8f5ff2352b61886fbc5e277bf65409ebae74fe 100644 --- a/plugin/v2/plugin.go +++ b/plugin/v2/plugin.go @@ -286,12 +286,12 @@ func (p *Plugin) GetRefCount() int { return p.refCount } -// SetRefCount sets the reference count. -func (p *Plugin) SetRefCount(count int) { +// AddRefCount adds to reference count. +func (p *Plugin) AddRefCount(count int) { p.mu.Lock() defer p.mu.Unlock() - p.refCount = count + p.refCount += count } // InitSpec creates an OCI spec from the plugin's config. From c027e4a86163f7d77b1e68bc856ca7f2762a9ec0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 13 Jun 2016 00:11:18 +0200 Subject: [PATCH 33/68] remove unsupported distros from install script The install script currently includes some distros that we don't actually have packages for. For these distros, the script currently performs step to install Docker from the distro's repository. This patch removes those distros from the install script, because we have no control over these packages, and cannot provide support for them. Installing docker anyway will give the false impression that they installed a package from our repository (but they didn't), and that they need to contact us for support. It's better to tell people that we don't install in that case, and refer them to the installation documentation, or the documentation of their distro. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 29b98b6ee6b7cd9a89bd8471032850a04427d471) Signed-off-by: Victor Vieux --- hack/install.sh | 95 ------------------------------------------------- 1 file changed, 95 deletions(-) diff --git a/hack/install.sh b/hack/install.sh index 6853ade8339c56d69e2d3a9e546feaf5394c6351..cc816324cf5e4824403d78412d86d6124acd72f2 100644 --- a/hack/install.sh +++ b/hack/install.sh @@ -348,74 +348,6 @@ do_install() { # Run setup for each distro accordingly case "$lsb_dist" in - amzn) - ( - set -x - $sh_c 'sleep 3; yum -y -q install docker' - ) - echo_docker_as_nonroot - exit 0 - ;; - - 'opensuse project'|opensuse) - echo 'Going to perform the following operations:' - if [ "$repo" != 'main' ]; then - echo ' * add repository obs://Virtualization:containers' - fi - echo ' * install Docker' - $sh_c 'echo "Press CTRL-C to abort"; sleep 3' - - if [ "$repo" != 'main' ]; then - # install experimental packages from OBS://Virtualization:containers - ( - set -x - zypper -n ar -f obs://Virtualization:containers Virtualization:containers - rpm_import_repository_key 55A0B34D49501BB7CA474F5AA193FBB572174FC2 - ) - fi - ( - set -x - zypper -n install docker - ) - echo_docker_as_nonroot - exit 0 - ;; - 'suse linux'|sle[sd]) - echo 'Going to perform the following operations:' - if [ "$repo" != 'main' ]; then - echo ' * add repository obs://Virtualization:containers' - echo ' * install experimental Docker using packages NOT supported by SUSE' - else - echo ' * add the "Containers" module' - echo ' * install Docker using packages supported by SUSE' - fi - $sh_c 'echo "Press CTRL-C to abort"; sleep 3' - - if [ "$repo" != 'main' ]; then - # install experimental packages from OBS://Virtualization:containers - echo >&2 'Warning: installing experimental packages from OBS, these packages are NOT supported by SUSE' - ( - set -x - zypper -n ar -f obs://Virtualization:containers/SLE_12 Virtualization:containers - rpm_import_repository_key 55A0B34D49501BB7CA474F5AA193FBB572174FC2 - ) - else - # Add the containers module - # Note well-1: the SLE machine must already be registered against SUSE Customer Center - # Note well-2: the `-r ""` is required to workaround a known issue of SUSEConnect - ( - set -x - SUSEConnect -p sle-module-containers/12/x86_64 -r "" - ) - fi - ( - set -x - zypper -n install docker - ) - echo_docker_as_nonroot - exit 0 - ;; - ubuntu|debian|raspbian) export DEBIAN_FRONTEND=noninteractive @@ -522,33 +454,6 @@ do_install() { echo_docker_as_nonroot exit 0 ;; - gentoo) - if [ "$url" = "https://test.docker.com/" ]; then - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-'EOF'", spaces are kept in the output - cat >&2 <<-'EOF' - - You appear to be trying to install the latest nightly build in Gentoo.' - The portage tree should contain the latest stable release of Docker, but' - if you want something more recent, you can always use the live ebuild' - provided in the "docker" overlay available via layman. For more' - instructions, please see the following URL:' - - https://github.com/tianon/docker-overlay#using-this-overlay' - - After adding the "docker" overlay, you should be able to:' - - emerge -av =app-emulation/docker-9999' - - EOF - exit 1 - fi - - ( - set -x - $sh_c 'sleep 3; emerge app-emulation/docker' - ) - exit 0 - ;; esac # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-'EOF'", spaces are kept in the output From 6e08e2edcf24f7f6b3598f2c7764946c4f1d365d Mon Sep 17 00:00:00 2001 From: Michael Friis Date: Mon, 5 Dec 2016 20:21:25 -0800 Subject: [PATCH 34/68] remove bonus whitespace Signed-off-by: Michael Friis (cherry picked from commit 8d47858f96c7ef03a9d3543e0994119390acb1bb) Signed-off-by: Victor Vieux --- hack/make.ps1 | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/hack/make.ps1 b/hack/make.ps1 index e9e04781bb9fa67b59f72d371704b80321edab2a..cb8bc7219886a1ac6b3fdbad41c2cf58e2f75492 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -6,17 +6,17 @@ by hack\make.sh, but uses native Windows PowerShell semantics. It does not support the full set of options provided by the Linux counterpart. For example: - + - You can't cross-build Linux docker binaries on Windows - Hashes aren't generated on binaries - 'Releasing' isn't supported. - Integration tests. This is because they currently cannot run inside a container, - and require significant external setup. - - It does however provided the minimum necessary to support parts of local Windows + and require significant external setup. + + It does however provided the minimum necessary to support parts of local Windows development and Windows to Windows CI. - Usage Examples (run from repo root): + Usage Examples (run from repo root): "hack\make.ps1 -Binary" to build the binaries "hack\make.ps1 -Client" to build just the client 64-bit binary "hack\make.ps1 -TestUnit" to run unit tests @@ -190,7 +190,7 @@ Function Validate-DCO($headCommit, $upstreamCommit) { # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :( $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{ $a=$_.Split(" "); $adds+=[int]$a[0]; $dels+=[int]$a[1] } - if (($adds -eq 0) -and ($dels -eq 0)) { + if (($adds -eq 0) -and ($dels -eq 0)) { Write-Warning "DCO validation - nothing to validate!" return } @@ -199,7 +199,7 @@ Function Validate-DCO($headCommit, $upstreamCommit) { if ($LASTEXITCODE -ne 0) { Throw "Failed git log --format" } $commits = $($commits -split '\s+' -match '\S') $badCommits=@() - $commits | %{ + $commits | %{ # Skip commits with no content such as merge commits etc if ($(git log -1 --format=format: --name-status $_).Length -gt 0) { # Ignore exit code on next call - always process regardless @@ -230,7 +230,7 @@ Function Validate-PkgImports($headCommit, $upstreamCommit) { # For the current changed file, get its list of dependencies, sorted and uniqued. $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file" if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" } - $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique + $imports = $imports -Replace "\[" -Replace "\]", "" -Split(" ") | Sort-Object | Get-Unique # Filter out what we are looking for $imports = $imports -NotMatch "^github.com/docker/docker/pkg/" ` -NotMatch "^github.com/docker/docker/vendor" ` @@ -255,11 +255,11 @@ Function Validate-GoFormat($headCommit, $upstreamCommit) { if ($(Get-Command gofmt -ErrorAction SilentlyContinue) -eq $nil) { Throw "gofmt does not appear to be installed" } # Get a list of all go source-code files which have changed. Ignore exit code on next call - always process regardless - $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'" + $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'*.go`'" $files = $files | Select-String -NotMatch "^vendor/" $badFiles=@(); $files | %{ # Deliberately ignore error on next line - treat as failed - $content=Invoke-Expression "git show $headCommit`:$_" + $content=Invoke-Expression "git show $headCommit`:$_" # Next set of hoops are to ensure we have LF not CRLF semantics as otherwise gofmt on Windows will not succeed. # Also note that gofmt on Windows does not appear to support stdin piping correctly. Hence go through a temporary file. @@ -327,7 +327,7 @@ Try { # Get the version of docker (eg 1.14.0-dev) $dockerVersion=Get-DockerVersion - # Give a warning if we are not running in a container and are building binaries or running unit tests. + # Give a warning if we are not running in a container and are building binaries or running unit tests. # Not relevant for validation tests as these are fine to run outside of a container. if ($Client -or $Daemon -or $TestUnit) { Check-InContainer } @@ -341,7 +341,7 @@ Try { Catch [Exception] { Throw $_ } } - # DCO, Package import and Go formatting tests. + # DCO, Package import and Go formatting tests. if ($DCO -or $PkgImports -or $GoFormat) { # We need the head and upstream commits for these $headCommit=Get-HeadCommit From b18516e6ac95c3adb4c9210ca92aa3dab1835b8d Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 7 Dec 2016 17:47:51 -0800 Subject: [PATCH 35/68] Windows: make.ps1 fix DCO check Signed-off-by: John Howard (cherry picked from commit e538c1fdca16cadf59f1d19df75857c8b2c4af06) Signed-off-by: Victor Vieux --- hack/make.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hack/make.ps1 b/hack/make.ps1 index cb8bc7219886a1ac6b3fdbad41c2cf58e2f75492..c085902ab65817368b54552ecaef5e552b624360 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -176,6 +176,7 @@ Function Execute-Build($type, $additionalBuildTags, $directory) { Pop-Location; $global:pushed=$False } + # Validates the DCO marker is present on each commit Function Validate-DCO($headCommit, $upstreamCommit) { Write-Host "INFO: Validating Developer Certificate of Origin..." @@ -189,8 +190,12 @@ Function Validate-DCO($headCommit, $upstreamCommit) { if ($LASTEXITCODE -ne 0) { Throw "Failed git diff --numstat" } # Counts of adds and deletes after removing multiple white spaces. AWK anyone? :( - $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{ $a=$_.Split(" "); $adds+=[int]$a[0]; $dels+=[int]$a[1] } - if (($adds -eq 0) -and ($dels -eq 0)) { + $adds=0; $dels=0; $($counts -replace '\s+', ' ') | %{ + $a=$_.Split(" "); + if ($a[0] -ne "-") { $adds+=[int]$a[0] } + if ($a[1] -ne "-") { $dels+=[int]$a[1] } + } + if (($adds -eq 0) -and ($dels -eq 0)) { Write-Warning "DCO validation - nothing to validate!" return } From 4fba52a2f7de3d1e46cb774490225faf131b7235 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 18 Aug 2016 14:23:10 -0700 Subject: [PATCH 36/68] Add registry-specific credential helper support Signed-off-by: Jake Sanders (cherry picked from commit 07c4b4124b46be30ea3ac7d114c44c4f911ca182) Signed-off-by: Victor Vieux --- cli/command/cli.go | 49 ++++++++++++-- cli/command/image/build.go | 4 +- cli/command/registry.go | 4 +- cli/command/registry/login.go | 2 +- cli/command/registry/logout.go | 2 +- cliconfig/config_test.go | 94 +++++++++++++++++++++++---- cliconfig/configfile/file.go | 4 +- cliconfig/credentials/native_store.go | 4 +- 8 files changed, 139 insertions(+), 24 deletions(-) diff --git a/cli/command/cli.go b/cli/command/cli.go index 99ea6331af626fcb37c41714acf8f1df557bbbf6..6d1dd7472ee062ba03458bbb61c91ecc079181e8 100644 --- a/cli/command/cli.go +++ b/cli/command/cli.go @@ -10,6 +10,7 @@ import ( "runtime" "github.com/docker/docker/api" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" cliflags "github.com/docker/docker/cli/flags" "github.com/docker/docker/cliconfig" @@ -86,15 +87,55 @@ func (cli *DockerCli) ConfigFile() *configfile.ConfigFile { return cli.configFile } +// GetAllCredentials returns all of the credentials stored in all of the +// configured credential stores. +func (cli *DockerCli) GetAllCredentials() (map[string]types.AuthConfig, error) { + auths := make(map[string]types.AuthConfig) + for registry := range cli.configFile.CredentialHelpers { + helper := cli.CredentialsStore(registry) + newAuths, err := helper.GetAll() + if err != nil { + return nil, err + } + addAll(auths, newAuths) + } + defaultStore := cli.CredentialsStore("") + newAuths, err := defaultStore.GetAll() + if err != nil { + return nil, err + } + addAll(auths, newAuths) + return auths, nil +} + +func addAll(to, from map[string]types.AuthConfig) { + for reg, ac := range from { + to[reg] = ac + } +} + // CredentialsStore returns a new credentials store based -// on the settings provided in the configuration file. -func (cli *DockerCli) CredentialsStore() credentials.Store { - if cli.configFile.CredentialsStore != "" { - return credentials.NewNativeStore(cli.configFile) +// on the settings provided in the configuration file. Empty string returns +// the default credential store. +func (cli *DockerCli) CredentialsStore(serverAddress string) credentials.Store { + if helper := getConfiguredCredentialStore(cli.configFile, serverAddress); helper != "" { + return credentials.NewNativeStore(cli.configFile, helper) } return credentials.NewFileStore(cli.configFile) } +// getConfiguredCredentialStore returns the credential helper configured for the +// given registry, the default credsStore, or the empty string if neither are +// configured. +func getConfiguredCredentialStore(c *configfile.ConfigFile, serverAddress string) string { + if c.CredentialHelpers != nil && serverAddress != "" { + if helper, exists := c.CredentialHelpers[serverAddress]; exists { + return helper + } + } + return c.CredentialsStore +} + // Initialize the dockerCli runs initialization that must happen after command // line flags are parsed. func (cli *DockerCli) Initialize(opts *cliflags.ClientOptions) error { diff --git a/cli/command/image/build.go b/cli/command/image/build.go index ebec87d64163a597f9080b3b23400bf0df044196..78cc41494f0cb1f97c19354338e533ca713a14ad 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -280,7 +280,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { } } - authConfig, _ := dockerCli.CredentialsStore().GetAll() + authConfigs, _ := dockerCli.GetAllCredentials() buildOptions := types.ImageBuildOptions{ Memory: memory, MemorySwap: memorySwap, @@ -301,7 +301,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { ShmSize: shmSize, Ulimits: options.ulimits.GetList(), BuildArgs: runconfigopts.ConvertKVStringsToMap(options.buildArgs.GetAll()), - AuthConfigs: authConfig, + AuthConfigs: authConfigs, Labels: runconfigopts.ConvertKVStringsToMap(options.labels.GetAll()), CacheFrom: options.cacheFrom, SecurityOpt: options.securityOpt, diff --git a/cli/command/registry.go b/cli/command/registry.go index b70d6f444c5a6fccd4755d94a93155e8a2105915..65f6b3309e783ec08ebd1e5a878ad260049e8059 100644 --- a/cli/command/registry.go +++ b/cli/command/registry.go @@ -67,7 +67,7 @@ func ResolveAuthConfig(ctx context.Context, cli *DockerCli, index *registrytypes configKey = ElectAuthServer(ctx, cli) } - a, _ := cli.CredentialsStore().Get(configKey) + a, _ := cli.CredentialsStore(configKey).Get(configKey) return a } @@ -82,7 +82,7 @@ func ConfigureAuth(cli *DockerCli, flUser, flPassword, serverAddress string, isD serverAddress = registry.ConvertToHostname(serverAddress) } - authconfig, err := cli.CredentialsStore().Get(serverAddress) + authconfig, err := cli.CredentialsStore(serverAddress).Get(serverAddress) if err != nil { return authconfig, err } diff --git a/cli/command/registry/login.go b/cli/command/registry/login.go index 93e1b40e36d97b4f5b1d9bf755ab13b5b859073a..05b3bb03b25b1369b5d96451b20ebd412fb5f555 100644 --- a/cli/command/registry/login.go +++ b/cli/command/registry/login.go @@ -74,7 +74,7 @@ func runLogin(dockerCli *command.DockerCli, opts loginOptions) error { authConfig.Password = "" authConfig.IdentityToken = response.IdentityToken } - if err := dockerCli.CredentialsStore().Store(authConfig); err != nil { + if err := dockerCli.CredentialsStore(serverAddress).Store(authConfig); err != nil { return fmt.Errorf("Error saving credentials: %v", err) } diff --git a/cli/command/registry/logout.go b/cli/command/registry/logout.go index 8e820dcc8c74785eefbb7a9c6189c06c6161afbf..877e60e8ccaff283c0b20d756b18e685a1a7ac86 100644 --- a/cli/command/registry/logout.go +++ b/cli/command/registry/logout.go @@ -68,7 +68,7 @@ func runLogout(dockerCli *command.DockerCli, serverAddress string) error { fmt.Fprintf(dockerCli.Out(), "Removing login credentials for %s\n", hostnameAddress) for _, r := range regsToLogout { - if err := dockerCli.CredentialsStore().Erase(r); err != nil { + if err := dockerCli.CredentialsStore(r).Erase(r); err != nil { fmt.Fprintf(dockerCli.Err(), "WARNING: could not erase credentials: %v\n", err) } } diff --git a/cliconfig/config_test.go b/cliconfig/config_test.go index 6eff26e4818a543905620a94cb8614f867e2db24..d8a099ab58e387463ae6b14617dfd9eaa91eea8a 100644 --- a/cliconfig/config_test.go +++ b/cliconfig/config_test.go @@ -86,7 +86,7 @@ func TestEmptyFile(t *testing.T) { } } -func TestEmptyJson(t *testing.T) { +func TestEmptyJSON(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -193,7 +193,7 @@ func TestOldValidAuth(t *testing.T) { } } -func TestOldJsonInvalid(t *testing.T) { +func TestOldJSONInvalid(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -219,7 +219,7 @@ func TestOldJsonInvalid(t *testing.T) { } } -func TestOldJson(t *testing.T) { +func TestOldJSON(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -265,7 +265,7 @@ func TestOldJson(t *testing.T) { } } -func TestNewJson(t *testing.T) { +func TestNewJSON(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -304,7 +304,7 @@ func TestNewJson(t *testing.T) { } } -func TestNewJsonNoEmail(t *testing.T) { +func TestNewJSONNoEmail(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -343,7 +343,7 @@ func TestNewJsonNoEmail(t *testing.T) { } } -func TestJsonWithPsFormat(t *testing.T) { +func TestJSONWithPsFormat(t *testing.T) { tmpHome, err := ioutil.TempDir("", "config-test") if err != nil { t.Fatal(err) @@ -376,6 +376,78 @@ func TestJsonWithPsFormat(t *testing.T) { } } +func TestJSONWithCredentialStore(t *testing.T) { + tmpHome, err := ioutil.TempDir("", "config-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpHome) + + fn := filepath.Join(tmpHome, ConfigFileName) + js := `{ + "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } }, + "credsStore": "crazy-secure-storage" +}` + if err := ioutil.WriteFile(fn, []byte(js), 0600); err != nil { + t.Fatal(err) + } + + config, err := Load(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + if config.CredentialsStore != "crazy-secure-storage" { + t.Fatalf("Unknown credential store: %s\n", config.CredentialsStore) + } + + // Now save it and make sure it shows up in new form + configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) + if !strings.Contains(configStr, `"credsStore":`) || + !strings.Contains(configStr, "crazy-secure-storage") { + t.Fatalf("Should have save in new form: %s", configStr) + } +} + +func TestJSONWithCredentialHelpers(t *testing.T) { + tmpHome, err := ioutil.TempDir("", "config-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpHome) + + fn := filepath.Join(tmpHome, ConfigFileName) + js := `{ + "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } }, + "credHelpers": { "images.io": "images-io", "containers.com": "crazy-secure-storage" } +}` + if err := ioutil.WriteFile(fn, []byte(js), 0600); err != nil { + t.Fatal(err) + } + + config, err := Load(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + if config.CredentialHelpers == nil { + t.Fatal("config.CredentialHelpers was nil") + } else if config.CredentialHelpers["images.io"] != "images-io" || + config.CredentialHelpers["containers.com"] != "crazy-secure-storage" { + t.Fatalf("Credential helpers not deserialized properly: %v\n", config.CredentialHelpers) + } + + // Now save it and make sure it shows up in new form + configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) + if !strings.Contains(configStr, `"credHelpers":`) || + !strings.Contains(configStr, "images.io") || + !strings.Contains(configStr, "images-io") || + !strings.Contains(configStr, "containers.com") || + !strings.Contains(configStr, "crazy-secure-storage") { + t.Fatalf("Should have save in new form: %s", configStr) + } +} + // Save it and make sure it shows up in new form func saveConfigAndValidateNewFormat(t *testing.T, config *configfile.ConfigFile, homeFolder string) string { if err := config.Save(); err != nil { @@ -420,7 +492,7 @@ func TestConfigFile(t *testing.T) { } } -func TestJsonReaderNoFile(t *testing.T) { +func TestJSONReaderNoFile(t *testing.T) { js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } } }` config, err := LoadFromReader(strings.NewReader(js)) @@ -435,7 +507,7 @@ func TestJsonReaderNoFile(t *testing.T) { } -func TestOldJsonReaderNoFile(t *testing.T) { +func TestOldJSONReaderNoFile(t *testing.T) { js := `{"https://index.docker.io/v1/":{"auth":"am9lam9lOmhlbGxv","email":"user@example.com"}}` config, err := LegacyLoadFromReader(strings.NewReader(js)) @@ -449,7 +521,7 @@ func TestOldJsonReaderNoFile(t *testing.T) { } } -func TestJsonWithPsFormatNoFile(t *testing.T) { +func TestJSONWithPsFormatNoFile(t *testing.T) { js := `{ "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } }, "psFormat": "table {{.ID}}\\t{{.Label \"com.docker.label.cpu\"}}" @@ -465,7 +537,7 @@ func TestJsonWithPsFormatNoFile(t *testing.T) { } -func TestJsonSaveWithNoFile(t *testing.T) { +func TestJSONSaveWithNoFile(t *testing.T) { js := `{ "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv" } }, "psFormat": "table {{.ID}}\\t{{.Label \"com.docker.label.cpu\"}}" @@ -507,7 +579,7 @@ func TestJsonSaveWithNoFile(t *testing.T) { } } -func TestLegacyJsonSaveWithNoFile(t *testing.T) { +func TestLegacyJSONSaveWithNoFile(t *testing.T) { js := `{"https://index.docker.io/v1/":{"auth":"am9lam9lOmhlbGxv","email":"user@example.com"}}` config, err := LegacyLoadFromReader(strings.NewReader(js)) diff --git a/cliconfig/configfile/file.go b/cliconfig/configfile/file.go index d7df229d29e881bc4f8c23014a146ea447727f99..39097133a42a68baddc2747f179a1ac3a6e4d7ea 100644 --- a/cliconfig/configfile/file.go +++ b/cliconfig/configfile/file.go @@ -31,6 +31,7 @@ type ConfigFile struct { StatsFormat string `json:"statsFormat,omitempty"` DetachKeys string `json:"detachKeys,omitempty"` CredentialsStore string `json:"credsStore,omitempty"` + CredentialHelpers map[string]string `json:"credHelpers,omitempty"` Filename string `json:"-"` // Note: for internal use only ServiceInspectFormat string `json:"serviceInspectFormat,omitempty"` } @@ -96,7 +97,8 @@ func (configFile *ConfigFile) LoadFromReader(configData io.Reader) error { // in this file or not. func (configFile *ConfigFile) ContainsAuth() bool { return configFile.CredentialsStore != "" || - (configFile.AuthConfigs != nil && len(configFile.AuthConfigs) > 0) + len(configFile.CredentialHelpers) > 0 || + len(configFile.AuthConfigs) > 0 } // SaveToWriter encodes and writes out all the authorization information to diff --git a/cliconfig/credentials/native_store.go b/cliconfig/credentials/native_store.go index 19a2beffc965c2153a7c97de2f3d2e3138dbb586..dec2dbcb822b382d847de6b4999f38fe4d595bde 100644 --- a/cliconfig/credentials/native_store.go +++ b/cliconfig/credentials/native_store.go @@ -22,8 +22,8 @@ type nativeStore struct { // NewNativeStore creates a new native store that // uses a remote helper program to manage credentials. -func NewNativeStore(file *configfile.ConfigFile) Store { - name := remoteCredentialsPrefix + file.CredentialsStore +func NewNativeStore(file *configfile.ConfigFile, helperSuffix string) Store { + name := remoteCredentialsPrefix + helperSuffix return &nativeStore{ programFunc: client.NewShellProgramFunc(name), fileStore: NewFileStore(file), From 64aac182d67228b9dc822469a2c697519b93e6a8 Mon Sep 17 00:00:00 2001 From: Doug Davis Date: Sat, 3 Dec 2016 05:46:04 -0800 Subject: [PATCH 37/68] Fix processing of unset build-args during build This reverts 26103. 26103 was trying to make it so that if someone did: docker build --build-arg FOO . and FOO wasn't set as an env var then it would pick-up FOO from the Dockerfile's ARG cmd. However, it went too far and removed the ability to specify a build arg w/o any value. Meaning it required the --build-arg param to always be in the form "name=value", and not just "name". This PR does the right fix - it allows just "name" and it'll grab the value from the env vars if set. If "name" isn't set in the env then it still needs to send "name" to the server so that a warning can be printed about an unused --build-arg. And this is why buildArgs in the options is now a *string instead of just a string - 'nil' == mentioned but no value. Closes #29084 Signed-off-by: Doug Davis (cherry picked from commit cdb8ea90b04683adb25c8ccd71b6eaedc44b51e2) Signed-off-by: Victor Vieux --- api/server/router/build/build_routes.go | 16 +++++- api/types/client.go | 11 ++-- builder/dockerfile/builder.go | 2 +- builder/dockerfile/dispatchers.go | 17 ++++--- builder/dockerfile/dispatchers_test.go | 4 +- builder/dockerfile/evaluator.go | 2 +- cli/command/image/build.go | 4 +- client/image_build_test.go | 11 ++-- integration-cli/docker_cli_build_test.go | 65 ++++++++++++++++++++++++ runconfig/opts/opts.go | 15 ------ runconfig/opts/opts_test.go | 36 ------------- runconfig/opts/parse.go | 19 +++++++ 12 files changed, 129 insertions(+), 73 deletions(-) diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go index e41c0a81eff9472cf8261e56aa61ae63a9cfe05b..75425b19fbe7bd3669be970bb27fa339ceaea763 100644 --- a/api/server/router/build/build_routes.go +++ b/api/server/router/build/build_routes.go @@ -84,14 +84,28 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.Ulimits = buildUlimits } - var buildArgs = map[string]string{} + var buildArgs = map[string]*string{} buildArgsJSON := r.FormValue("buildargs") + + // Note that there are two ways a --build-arg might appear in the + // json of the query param: + // "foo":"bar" + // and "foo":nil + // The first is the normal case, ie. --build-arg foo=bar + // or --build-arg foo + // where foo's value was picked up from an env var. + // The second ("foo":nil) is where they put --build-arg foo + // but "foo" isn't set as an env var. In that case we can't just drop + // the fact they mentioned it, we need to pass that along to the builder + // so that it can print a warning about "foo" being unused if there is + // no "ARG foo" in the Dockerfile. if buildArgsJSON != "" { if err := json.Unmarshal([]byte(buildArgsJSON), &buildArgs); err != nil { return nil, err } options.BuildArgs = buildArgs } + var labels = map[string]string{} labelsJSON := r.FormValue("labels") if labelsJSON != "" { diff --git a/api/types/client.go b/api/types/client.go index 850e19e6e3735b86818ac2f0391a7d871441d7ef..d58a1a10ed59b7c2c71adb16aaeac32af0829bbe 100644 --- a/api/types/client.go +++ b/api/types/client.go @@ -160,10 +160,13 @@ type ImageBuildOptions struct { ShmSize int64 Dockerfile string Ulimits []*units.Ulimit - BuildArgs map[string]string - AuthConfigs map[string]AuthConfig - Context io.Reader - Labels map[string]string + // See the parsing of buildArgs in api/server/router/build/build_routes.go + // for an explaination of why BuildArgs needs to use *string instead of + // just a string + BuildArgs map[string]*string + AuthConfigs map[string]AuthConfig + Context io.Reader + Labels map[string]string // squash the resulting image's layers to the parent // preserves the original image and creates a new one from the parent with all // the changes applied to a single layer diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index d13fe184f56e28f7db37fbe374373652b1ac92cd..da43513fff3681ced5c837e4cf8a25172c014028 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -125,7 +125,7 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back config = new(types.ImageBuildOptions) } if config.BuildArgs == nil { - config.BuildArgs = make(map[string]string) + config.BuildArgs = make(map[string]*string) } ctx, cancel := context.WithCancel(clientCtx) b = &Builder{ diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go index cf69fa496ad41da9df63cfc5e66d0ee60c4953fd..4f3dcd8cd33b66cbf0b1ce1309b0e3adf9b6001e 100644 --- a/builder/dockerfile/dispatchers.go +++ b/builder/dockerfile/dispatchers.go @@ -384,8 +384,8 @@ func run(b *Builder, args []string, attributes map[string]bool, original string) // the entire file (see 'leftoverArgs' processing in evaluator.go ) continue } - if _, ok := configEnv[key]; !ok { - cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, val)) + if _, ok := configEnv[key]; !ok && val != nil { + cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, *val)) } } @@ -728,7 +728,7 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string) var ( name string - value string + newValue string hasDefault bool ) @@ -745,7 +745,7 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string) } name = parts[0] - value = parts[1] + newValue = parts[1] hasDefault = true } else { name = arg @@ -756,9 +756,12 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string) // If there is a default value associated with this arg then add it to the // b.buildArgs if one is not already passed to the builder. The args passed - // to builder override the default value of 'arg'. - if _, ok := b.options.BuildArgs[name]; !ok && hasDefault { - b.options.BuildArgs[name] = value + // to builder override the default value of 'arg'. Note that a 'nil' for + // a value means that the user specified "--build-arg FOO" and "FOO" wasn't + // defined as an env var - and in that case we DO want to use the default + // value specified in the ARG cmd. + if baValue, ok := b.options.BuildArgs[name]; (!ok || baValue == nil) && hasDefault { + b.options.BuildArgs[name] = &newValue } return b.commit("", b.runConfig.Cmd, fmt.Sprintf("ARG %s", arg)) diff --git a/builder/dockerfile/dispatchers_test.go b/builder/dockerfile/dispatchers_test.go index a57f78a0e14c35629b95e2ca4610ba48057e369c..f7c57f7e3b837f8d39c4ae1adc592689de5eb177 100644 --- a/builder/dockerfile/dispatchers_test.go +++ b/builder/dockerfile/dispatchers_test.go @@ -460,7 +460,7 @@ func TestStopSignal(t *testing.T) { } func TestArg(t *testing.T) { - buildOptions := &types.ImageBuildOptions{BuildArgs: make(map[string]string)} + buildOptions := &types.ImageBuildOptions{BuildArgs: make(map[string]*string)} b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true, allowedBuildArgs: make(map[string]bool), options: buildOptions} @@ -488,7 +488,7 @@ func TestArg(t *testing.T) { t.Fatalf("%s argument should be a build arg", argName) } - if val != "bar" { + if *val != "bar" { t.Fatalf("%s argument should have default value 'bar', got %s", argName, val) } } diff --git a/builder/dockerfile/evaluator.go b/builder/dockerfile/evaluator.go index c3843936f21c0b45ba8e11794607967e8d7be3ee..f5997c91a6d88ec0ad3b6a940f5c2447ee38f260 100644 --- a/builder/dockerfile/evaluator.go +++ b/builder/dockerfile/evaluator.go @@ -158,7 +158,7 @@ func (b *Builder) dispatch(stepN int, stepTotal int, ast *parser.Node) error { // the entire file (see 'leftoverArgs' processing in evaluator.go ) continue } - envs = append(envs, fmt.Sprintf("%s=%s", key, val)) + envs = append(envs, fmt.Sprintf("%s=%s", key, *val)) } for ast.Next != nil { ast = ast.Next diff --git a/cli/command/image/build.go b/cli/command/image/build.go index 78cc41494f0cb1f97c19354338e533ca713a14ad..1699b2c45c02a1369cc9db08b6f321be4532fa54 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -67,7 +67,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command { ulimits := make(map[string]*units.Ulimit) options := buildOptions{ tags: opts.NewListOpts(validateTag), - buildArgs: opts.NewListOpts(runconfigopts.ValidateArg), + buildArgs: opts.NewListOpts(runconfigopts.ValidateEnv), ulimits: runconfigopts.NewUlimitOpt(&ulimits), labels: opts.NewListOpts(runconfigopts.ValidateEnv), } @@ -300,7 +300,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error { Dockerfile: relDockerfile, ShmSize: shmSize, Ulimits: options.ulimits.GetList(), - BuildArgs: runconfigopts.ConvertKVStringsToMap(options.buildArgs.GetAll()), + BuildArgs: runconfigopts.ConvertKVStringsToMapWithNil(options.buildArgs.GetAll()), AuthConfigs: authConfigs, Labels: runconfigopts.ConvertKVStringsToMap(options.labels.GetAll()), CacheFrom: options.cacheFrom, diff --git a/client/image_build_test.go b/client/image_build_test.go index ec0cbe2ee44f013a44d76069ba1c8f07aeaef37d..b9d04f817acce0e90725bcb1d6ed076584559801 100644 --- a/client/image_build_test.go +++ b/client/image_build_test.go @@ -27,6 +27,8 @@ func TestImageBuildError(t *testing.T) { } func TestImageBuild(t *testing.T) { + v1 := "value1" + v2 := "value2" emptyRegistryConfig := "bnVsbA==" buildCases := []struct { buildOptions types.ImageBuildOptions @@ -105,13 +107,14 @@ func TestImageBuild(t *testing.T) { }, { buildOptions: types.ImageBuildOptions{ - BuildArgs: map[string]string{ - "ARG1": "value1", - "ARG2": "value2", + BuildArgs: map[string]*string{ + "ARG1": &v1, + "ARG2": &v2, + "ARG3": nil, }, }, expectedQueryParams: map[string]string{ - "buildargs": `{"ARG1":"value1","ARG2":"value2"}`, + "buildargs": `{"ARG1":"value1","ARG2":"value2","ARG3":null}`, "rm": "0", }, expectedTags: []string{}, diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index e3524c7bd9d129e08a271e7f1a23603b368227d0..c05e545b016a0a732217801de457858d91737fac 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -6070,6 +6070,71 @@ func (s *DockerSuite) TestBuildBuildTimeArgUnconsumedArg(c *check.C) { } +func (s *DockerSuite) TestBuildBuildTimeArgEnv(c *check.C) { + testRequires(c, DaemonIsLinux) // Windows does not support ARG + args := []string{ + "build", + "--build-arg", fmt.Sprintf("FOO1=fromcmd"), + "--build-arg", fmt.Sprintf("FOO2="), + "--build-arg", fmt.Sprintf("FOO3"), // set in env + "--build-arg", fmt.Sprintf("FOO4"), // not set in env + "--build-arg", fmt.Sprintf("FOO5=fromcmd"), + // FOO6 is not set at all + "--build-arg", fmt.Sprintf("FOO7=fromcmd"), // should produce a warning + "--build-arg", fmt.Sprintf("FOO8="), // should produce a warning + "--build-arg", fmt.Sprintf("FOO9"), // should produce a warning + ".", + } + + dockerfile := fmt.Sprintf(`FROM busybox + ARG FOO1=fromfile + ARG FOO2=fromfile + ARG FOO3=fromfile + ARG FOO4=fromfile + ARG FOO5 + ARG FOO6 + RUN env + RUN [ "$FOO1" == "fromcmd" ] + RUN [ "$FOO2" == "" ] + RUN [ "$FOO3" == "fromenv" ] + RUN [ "$FOO4" == "fromfile" ] + RUN [ "$FOO5" == "fromcmd" ] + # The following should not exist at all in the env + RUN [ "$(env | grep FOO6)" == "" ] + RUN [ "$(env | grep FOO7)" == "" ] + RUN [ "$(env | grep FOO8)" == "" ] + RUN [ "$(env | grep FOO9)" == "" ] + `) + + ctx, err := fakeContext(dockerfile, nil) + c.Assert(err, check.IsNil) + defer ctx.Close() + + cmd := exec.Command(dockerBinary, args...) + cmd.Dir = ctx.Dir + cmd.Env = append(os.Environ(), + "FOO1=fromenv", + "FOO2=fromenv", + "FOO3=fromenv") + out, _, err := runCommandWithOutput(cmd) + if err != nil { + c.Fatal(err, out) + } + + // Now check to make sure we got a warning msg about unused build-args + i := strings.Index(out, "[Warning]") + if i < 0 { + c.Fatalf("Missing the build-arg warning in %q", out) + } + + out = out[i:] // "out" should contain just the warning message now + + // These were specified on a --build-arg but no ARG was in the Dockerfile + c.Assert(out, checker.Contains, "FOO7") + c.Assert(out, checker.Contains, "FOO8") + c.Assert(out, checker.Contains, "FOO9") +} + func (s *DockerSuite) TestBuildBuildTimeArgQuotedValVariants(c *check.C) { imgName := "bldargtest" envKey := "foo" diff --git a/runconfig/opts/opts.go b/runconfig/opts/opts.go index 2abce5744b36c13833f6f6b9a18879ebe2c805e4..cae0432f05d6f88c66b3f2867230d9214e376f53 100644 --- a/runconfig/opts/opts.go +++ b/runconfig/opts/opts.go @@ -59,21 +59,6 @@ func doesEnvExist(name string) bool { return false } -// ValidateArg validates a build-arg variable and returns it. -// Build-arg is in the form of = where is required. -func ValidateArg(val string) (string, error) { - arr := strings.Split(val, "=") - if len(arr) > 1 && isNotEmpty(arr[0]) { - return val, nil - } - - return "", fmt.Errorf("bad format for build-arg: %s", val) -} - -func isNotEmpty(val string) bool { - return len(val) > 0 -} - // ValidateExtraHost validates that the specified string is a valid extrahost and returns it. // ExtraHost is in the form of name:ip where the ip has to be a valid ip (IPv4 or IPv6). func ValidateExtraHost(val string) (string, error) { diff --git a/runconfig/opts/opts_test.go b/runconfig/opts/opts_test.go index 7bf9d2672a0bd0d10926a93a4b9f4d0619223945..43f8730fc4c256503dd5988c9c14a03daf5aaefd 100644 --- a/runconfig/opts/opts_test.go +++ b/runconfig/opts/opts_test.go @@ -66,42 +66,6 @@ func TestValidateEnv(t *testing.T) { } } -func TestValidateArg(t *testing.T) { - valids := map[string]string{ - "_=a": "_=a", - "var1=value1": "var1=value1", - "_var1=value1": "_var1=value1", - "var2=value2=value3": "var2=value2=value3", - "var3=abc!qwe": "var3=abc!qwe", - "var_4=value 4": "var_4=value 4", - "var_5=": "var_5=", - } - for value, expected := range valids { - actual, err := ValidateArg(value) - if err != nil { - t.Fatal(err) - } - if actual != expected { - t.Fatalf("Expected [%v], got [%v]", expected, actual) - } - } - - invalid := map[string]string{ - "foo": "bad format", - "=foo": "bad format", - "cc c": "bad format", - } - for value, expectedError := range invalid { - if _, err := ValidateArg(value); err == nil { - t.Fatalf("ValidateArg(`%s`) should have failed validation", value) - } else { - if !strings.Contains(err.Error(), expectedError) { - t.Fatalf("ValidateArg(`%s`) error should contain %q", value, expectedError) - } - } - } -} - func TestValidateExtraHosts(t *testing.T) { valid := []string{ `myhost:192.168.0.1`, diff --git a/runconfig/opts/parse.go b/runconfig/opts/parse.go index c3aa6e98b187c911b067f252a1b2be20130ed5a8..71a89277ec1c7b73016d79e3753688cd38cd78cb 100644 --- a/runconfig/opts/parse.go +++ b/runconfig/opts/parse.go @@ -705,6 +705,25 @@ func ConvertKVStringsToMap(values []string) map[string]string { return result } +// ConvertKVStringsToMapWithNil converts ["key=value"] to {"key":"value"} +// but set unset keys to nil - meaning the ones with no "=" in them. +// We use this in cases where we need to distinguish between +// FOO= and FOO +// where the latter case just means FOO was mentioned but not given a value +func ConvertKVStringsToMapWithNil(values []string) map[string]*string { + result := make(map[string]*string, len(values)) + for _, value := range values { + kv := strings.SplitN(value, "=", 2) + if len(kv) == 1 { + result[kv[0]] = nil + } else { + result[kv[0]] = &kv[1] + } + } + + return result +} + func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { loggingOptsMap := ConvertKVStringsToMap(loggingOpts) if loggingDriver == "none" && len(loggingOpts) > 0 { From 870beb70fb074ad78bacd5cac50bad854766ad29 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 5 Dec 2016 08:56:42 -0800 Subject: [PATCH 38/68] xFix issue for `--hostname` when running in "--net=host" This fix tries to address the issue raised in 29129 where "--hostname" not working when running in "--net=host" for `docker run`. The fix fixes the issue by not resetting the `container.Config.Hostname` if the `Hostname` has already been assigned through `--hostname`. An integration test has been added to cover the changes. This fix fixes 29129. Signed-off-by: Yong Tang (cherry picked from commit b0a7b0120f4461daa34527a743087e73ef8f5963) Signed-off-by: Victor Vieux --- daemon/container.go | 14 ++++++++++++-- daemon/container_operations.go | 8 +++++--- daemon/create.go | 2 +- integration-cli/docker_cli_run_test.go | 22 ++++++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/daemon/container.go b/daemon/container.go index dd4a5d6c16944d7cbe0844c2ba2c3c17578dc701..2a44800098d3b7c821c4638b932a203bfb77a1e7 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "os" "path/filepath" "time" @@ -101,7 +102,7 @@ func (daemon *Daemon) Register(c *container.Container) error { return nil } -func (daemon *Daemon) newContainer(name string, config *containertypes.Config, imgID image.ID, managed bool) (*container.Container, error) { +func (daemon *Daemon) newContainer(name string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) { var ( id string err error @@ -112,7 +113,16 @@ func (daemon *Daemon) newContainer(name string, config *containertypes.Config, i return nil, err } - daemon.generateHostname(id, config) + if hostConfig.NetworkMode.IsHost() { + if config.Hostname == "" { + config.Hostname, err = os.Hostname() + if err != nil { + return nil, err + } + } + } else { + daemon.generateHostname(id, config) + } entrypoint, args := daemon.getEntrypointAndArgs(config.Entrypoint, config.Cmd) base := daemon.newBaseContainer(id) diff --git a/daemon/container_operations.go b/daemon/container_operations.go index 5224308b7c3fc3a8c7251aa0fb59b83209c96f75..c30250622da0b35f1f8472b3a68559abde058eb9 100644 --- a/daemon/container_operations.go +++ b/daemon/container_operations.go @@ -851,9 +851,11 @@ func (daemon *Daemon) initializeNetworking(container *container.Container) error } if container.HostConfig.NetworkMode.IsHost() { - container.Config.Hostname, err = os.Hostname() - if err != nil { - return err + if container.Config.Hostname == "" { + container.Config.Hostname, err = os.Hostname() + if err != nil { + return err + } } } diff --git a/daemon/create.go b/daemon/create.go index 85d3779ebb84e63294b90d7794b47281c44dd6b8..c71d14e5fc1caa019318c296d265834c48805a58 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -96,7 +96,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( return nil, err } - if container, err = daemon.newContainer(params.Name, params.Config, imgID, managed); err != nil { + if container, err = daemon.newContainer(params.Name, params.Config, params.HostConfig, imgID, managed); err != nil { return nil, err } defer func() { diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 62b3a8d375359ba1d808e4439529fff519087d8f..58aec393c807f4cb2ab97e3db99a1e0334f5edd3 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -4665,3 +4665,25 @@ func (s *delayedReader) Read([]byte) (int, error) { time.Sleep(500 * time.Millisecond) return 0, io.EOF } + +// #28823 (originally #28639) +func (s *DockerSuite) TestRunMountReadOnlyDevShm(c *check.C) { + testRequires(c, SameHostDaemon, DaemonIsLinux) + emptyDir, err := ioutil.TempDir("", "test-read-only-dev-shm") + c.Assert(err, check.IsNil) + defer os.RemoveAll(emptyDir) + out, _, err := dockerCmdWithError("run", "--rm", "--read-only", + "-v", fmt.Sprintf("%s:/dev/shm:ro", emptyDir), + "busybox", "touch", "/dev/shm/foo") + c.Assert(err, checker.NotNil, check.Commentf(out)) + c.Assert(out, checker.Contains, "Read-only file system") +} + +// Test case for 29129 +func (s *DockerSuite) TestRunHostnameInHostMode(c *check.C) { + testRequires(c, DaemonIsLinux) + + expectedOutput := "foobar\nfoobar" + out, _ := dockerCmd(c, "run", "--net=host", "--hostname=foobar", "busybox", "sh", "-c", `echo $HOSTNAME && hostname`) + c.Assert(strings.TrimSpace(out), checker.Equals, expectedOutput) +} From 159f4f14d71dd7f3fced7b620e61ada2f88166e0 Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Mon, 12 Dec 2016 17:24:12 +0800 Subject: [PATCH 39/68] replace env description with args in extend config Signed-off-by: lixiaobing10051267 (cherry picked from commit abdc031aeaa757546735da0075868b87dd99ef7a) Signed-off-by: Victor Vieux --- docs/extend/config.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/extend/config.md b/docs/extend/config.md index e068eaccfaf141d4ba38b0ec2d063258452337c0..93ed4958a3e4b485c0f40942fcfbb31dc4533643 100644 --- a/docs/extend/config.md +++ b/docs/extend/config.md @@ -137,11 +137,11 @@ Config provides the base accessible fields for working with V0 plugin format - **`name`** *string* - name of the env. + name of the args. - **`description`** *string* - description of the env. + description of the args. - **`value`** *string array* From 508d06752d530c052e1fa028e600f5dfeaaa42b1 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Mon, 12 Dec 2016 09:14:10 +0100 Subject: [PATCH 40/68] Add zsh completion for 'docker inspect --type=plugin' and other missing ones Signed-off-by: Steve Durrheimer (cherry picked from commit 2bdffc1fb5f66233fe23410de17a6c5e53cfe8ef) Signed-off-by: Victor Vieux --- contrib/completion/zsh/_docker | 56 ++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 3afcb8977c51009aa764ae09b3352def66a51069..229a0edb7fe05657fce91a1cc74621515426a4fb 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -348,14 +348,14 @@ __docker_complete_ps_filters() { __docker_complete_containers_names && ret=0 ;; (network) - __docker_networks && ret=0 + __docker_complete_networks && ret=0 ;; (status) status_opts=('created' 'dead' 'exited' 'paused' 'restarting' 'running' 'removing') _describe -t status-filter-opts "status filter options" status_opts && ret=0 ;; (volume) - __docker_volumes && ret=0 + __docker_complete_volumes && ret=0 ;; *) _message 'value' && ret=0 @@ -453,7 +453,7 @@ __docker_complete_events_filter() { __docker_complete_images && ret=0 ;; (network) - __docker_networks && ret=0 + __docker_complete_networks && ret=0 ;; (type) local -a type_opts @@ -461,7 +461,7 @@ __docker_complete_events_filter() { _describe -t type-filter-opts "type filter options" type_opts && ret=0 ;; (volume) - __docker_volumes && ret=0 + __docker_complete_volumes && ret=0 ;; *) _message 'value' && ret=0 @@ -1033,10 +1033,10 @@ __docker_network_complete_ls_filters() { __docker_complete_info_plugins Network && ret=0 ;; (id) - __docker_networks_ids && ret=0 + __docker_complete_networks_ids && ret=0 ;; (name) - __docker_networks_names && ret=0 + __docker_complete_networks_names && ret=0 ;; (type) type_opts=('builtin' 'custom') @@ -1082,6 +1082,7 @@ __docker_get_networks() { for line in $lines; do s="${line[${begin[NETWORK ID]},${end[NETWORK ID]}]%% ##}" s="$s:${(l:7:: :::)${${line[${begin[DRIVER]},${end[DRIVER]}]}%% ##}}" + s="$s, ${${line[${begin[SCOPE]},${end[SCOPE]}]}%% ##}" networks=($networks $s) done fi @@ -1091,6 +1092,7 @@ __docker_get_networks() { for line in $lines; do s="${line[${begin[NAME]},${end[NAME]}]%% ##}" s="$s:${(l:7:: :::)${${line[${begin[DRIVER]},${end[DRIVER]}]}%% ##}}" + s="$s, ${${line[${begin[SCOPE]},${end[SCOPE]}]}%% ##}" networks=($networks $s) done fi @@ -1099,17 +1101,17 @@ __docker_get_networks() { return ret } -__docker_networks() { +__docker_complete_networks() { [[ $PREFIX = -* ]] && return 1 __docker_get_networks all "$@" } -__docker_networks_ids() { +__docker_complete_networks_ids() { [[ $PREFIX = -* ]] && return 1 __docker_get_networks ids "$@" } -__docker_networks_names() { +__docker_complete_networks_names() { [[ $PREFIX = -* ]] && return 1 __docker_get_networks names "$@" } @@ -1144,7 +1146,7 @@ __docker_network_subcommand() { "($help)--ip6=[Container IPv6 address]:IPv6: " \ "($help)*--link=[Add a link to another container]:link:->link" \ "($help)*--link-local-ip=[Add a link-local address for the container]:IPv4/IPv6: " \ - "($help -)1:network:__docker_networks" \ + "($help -)1:network:__docker_complete_networks" \ "($help -)2:containers:__docker_complete_containers" && ret=0 case $state in @@ -1177,14 +1179,14 @@ __docker_network_subcommand() { (disconnect) _arguments $(__docker_arguments) \ $opts_help \ - "($help -)1:network:__docker_networks" \ + "($help -)1:network:__docker_complete_networks" \ "($help -)2:containers:__docker_complete_containers" && ret=0 ;; (inspect) _arguments $(__docker_arguments) \ $opts_help \ "($help -f --format)"{-f=,--format=}"[Format the output using the given go template]:template: " \ - "($help -)*:network:__docker_networks" && ret=0 + "($help -)*:network:__docker_complete_networks" && ret=0 ;; (ls) _arguments $(__docker_arguments) \ @@ -1207,7 +1209,7 @@ __docker_network_subcommand() { (rm) _arguments $(__docker_arguments) \ $opts_help \ - "($help -)*:network:__docker_networks" && ret=0 + "($help -)*:network:__docker_complete_networks" && ret=0 ;; (help) _arguments $(__docker_arguments) ":subcommand:__docker_network_commands" && ret=0 @@ -2168,7 +2170,7 @@ __docker_volume_complete_ls_filters() { __docker_complete_info_plugins Volume && ret=0 ;; (name) - __docker_volumes && ret=0 + __docker_complete_volumes && ret=0 ;; *) _message 'value' && ret=0 @@ -2182,7 +2184,7 @@ __docker_volume_complete_ls_filters() { return ret } -__docker_volumes() { +__docker_complete_volumes() { [[ $PREFIX = -* ]] && return 1 integer ret=1 declare -a lines volumes @@ -2246,7 +2248,7 @@ __docker_volume_subcommand() { _arguments $(__docker_arguments) \ $opts_help \ "($help -f --format)"{-f=,--format=}"[Format the output using the given go template]:template: " \ - "($help -)1:volume:__docker_volumes" && ret=0 + "($help -)1:volume:__docker_complete_volumes" && ret=0 ;; (ls) _arguments $(__docker_arguments) \ @@ -2269,7 +2271,7 @@ __docker_volume_subcommand() { _arguments $(__docker_arguments) \ $opts_help \ "($help -f --force)"{-f,--force}"[Force the removal of one or more volumes]" \ - "($help -):volume:__docker_volumes" && ret=0 + "($help -):volume:__docker_complete_volumes" && ret=0 ;; (help) _arguments $(__docker_arguments) ":subcommand:__docker_volume_commands" && ret=0 @@ -2458,7 +2460,7 @@ __docker_subcommand() { $opts_help \ "($help -f --format)"{-f=,--format=}"[Format the output using the given go template]:template: " \ "($help -s --size)"{-s,--size}"[Display total file sizes if the type is container]" \ - "($help)--type=[Return JSON for specified type]:type:(image container)" \ + "($help)--type=[Return JSON for specified type]:type:(container image network node plugin service volume)" \ "($help -)*: :->values" && ret=0 case $state in @@ -2467,8 +2469,24 @@ __docker_subcommand() { __docker_complete_containers && ret=0 elif [[ ${words[(r)--type=image]} == --type=image ]]; then __docker_complete_images && ret=0 + elif [[ ${words[(r)--type=network]} == --type=network ]]; then + __docker_complete_networks && ret=0 + elif [[ ${words[(r)--type=node]} == --type=node ]]; then + __docker_complete_nodes && ret=0 + elif [[ ${words[(r)--type=plugin]} == --type=plugin ]]; then + __docker_complete_plugins && ret=0 + elif [[ ${words[(r)--type=service]} == --type=service ]]; then + __docker_complete_services && ret=0 + elif [[ ${words[(r)--type=volume]} == --type=volume ]]; then + __docker_complete_volumes && ret=0 else - __docker_complete_images && __docker_complete_containers && ret=0 + __docker_complete_containers + __docker_complete_images + __docker_complete_networks + __docker_complete_nodes + __docker_complete_plugins + __docker_complete_services + __docker_complete_volumes && ret=0 fi ;; esac From 0d575d1304b2211867c3e936448bedd4314a8665 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Sun, 11 Dec 2016 16:37:53 +0100 Subject: [PATCH 41/68] Fix link to volume label support in changelog Signed-off-by: Martin Honermeyer (cherry picked from commit bb06e138e47ff2a7b1b3553a6337a2bb64b58907) Signed-off-by: Victor Vieux --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aca0eec1896d6498dd3721c9ad279370fdc0627f..11f7a3d3e8ea61bb45b9afe4a9a8b8796ffddd83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -179,7 +179,7 @@ be found. ### Volume -+ Add support for labels on volumes [#25628](https://github.com/docker/docker/pull/21567) ++ Add support for labels on volumes [#21270](https://github.com/docker/docker/pull/21270) + Add support for filtering volumes by label [#25628](https://github.com/docker/docker/pull/25628) * Add a `--force` flag in `docker volume rm` to forcefully purge the data of the volume that has already been deleted [#23436](https://github.com/docker/docker/pull/23436) * Enhance `docker volume inspect` to show all options used when creating the volume [#26671](https://github.com/docker/docker/pull/26671) From 9a6c5e14bed7e48bfa18a8b15d16f1b38828ddee Mon Sep 17 00:00:00 2001 From: Kei Ohmura Date: Sun, 11 Dec 2016 21:21:31 +0900 Subject: [PATCH 42/68] docs: fix description of `docker swarm join --help` Signed-off-by: Kei Ohmura (cherry picked from commit 77dd8474a7b4447d7c5b1d257afe1bb2f6443172) Signed-off-by: Victor Vieux --- docs/reference/commandline/swarm_join.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/commandline/swarm_join.md b/docs/reference/commandline/swarm_join.md index 7753a487310e52e076ec6943d9d6dc0cab0bf012..0cde0d7bcd8aeba9992c38472261988b424e838a 100644 --- a/docs/reference/commandline/swarm_join.md +++ b/docs/reference/commandline/swarm_join.md @@ -21,10 +21,10 @@ Usage: docker swarm join [OPTIONS] HOST:PORT Join a swarm as a node and/or manager Options: - --advertise-addr value Advertised address (format: [:port]) - --help Print usage - --listen-addr value Listen address (format: [:port) - --token string Token for entry into the swarm + --advertise-addr string Advertised address (format: [:port]) + --help Print usage + --listen-addr node-addr Listen address (format: [:port]) (default 0.0.0.0:2377) + --token string Token for entry into the swarm ``` Join a node to a swarm. The node joins as a manager node or worker node based upon the token you From cc855179cfa81ca3610c440dd656f1b5c8fab2fe Mon Sep 17 00:00:00 2001 From: Kei Ohmura Date: Sun, 11 Dec 2016 21:25:40 +0900 Subject: [PATCH 43/68] docs: fix description of `docker swarm update --help` Signed-off-by: Kei Ohmura (cherry picked from commit 2f0e00f58723cb3e063b49f564173b3768476c99) Signed-off-by: Victor Vieux --- docs/reference/commandline/swarm_update.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/commandline/swarm_update.md b/docs/reference/commandline/swarm_update.md index 84b79135a3e8004eee78943bb0d4ea3470c62416..0af63fe3e080bedf630d129158f0996948026d98 100644 --- a/docs/reference/commandline/swarm_update.md +++ b/docs/reference/commandline/swarm_update.md @@ -24,10 +24,10 @@ Options: --autolock Change manager autolocking setting (true|false) --cert-expiry duration Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s) --dispatcher-heartbeat duration Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s) - --external-ca value Specifications of one or more certificate signing endpoints + --external-ca external-ca Specifications of one or more certificate signing endpoints --help Print usage - --max-snapshots int Number of additional Raft snapshots to retain - --snapshot-interval int Number of log entries between Raft snapshots + --max-snapshots uint Number of additional Raft snapshots to retain + --snapshot-interval uint Number of log entries between Raft snapshots (default 10000) --task-history-limit int Task history retention limit (default 5) ``` From fcb58702bcfb443865f4ca34eeef99089defa828 Mon Sep 17 00:00:00 2001 From: Qinglan Peng Date: Sun, 11 Dec 2016 16:13:13 +0800 Subject: [PATCH 44/68] fix some broken contents links Signed-off-by: Qinglan Peng fix-contents-links Signed-off-by: Qinglan Peng (cherry picked from commit bac792c1634724def05c5ebc93891ecd5b353a53) Signed-off-by: Victor Vieux --- docs/reference/run.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/run.md b/docs/reference/run.md index 909e58e2f927696e1124b7e6fb4f93e71a72d3d6..73769ed6104e7a64752d369049137c934acfa8f1 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -63,15 +63,15 @@ Only the operator (the person executing `docker run`) can set the following options. - [Detached vs foreground](#detached-vs-foreground) - - [Detached (-d)](#detached-d) + - [Detached (-d)](#detached--d) - [Foreground](#foreground) - [Container identification](#container-identification) - - [Name (--name)](#name-name) + - [Name (--name)](#name---name) - [PID equivalent](#pid-equivalent) - - [IPC settings (--ipc)](#ipc-settings-ipc) + - [IPC settings (--ipc)](#ipc-settings---ipc) - [Network settings](#network-settings) - - [Restart policies (--restart)](#restart-policies-restart) - - [Clean up (--rm)](#clean-up-rm) + - [Restart policies (--restart)](#restart-policies---restart) + - [Clean up (--rm)](#clean-up---rm) - [Runtime constraints on resources](#runtime-constraints-on-resources) - [Runtime privilege and Linux capabilities](#runtime-privilege-and-linux-capabilities) From 85ae13f6d0c77fc23340ee779fc8fcd97f304440 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sat, 10 Dec 2016 13:30:45 -0800 Subject: [PATCH 45/68] Add bash completion for `docker inspect --type plugin` Signed-off-by: Harald Albers (cherry picked from commit 81b4b2b5fa098756149aa3f1c44715c5ccc08860) Signed-off-by: Victor Vieux --- contrib/completion/bash/docker | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index efaffdc509993fcc3f97297cb453ac5d01b1fdd2..7f9a13f71041a53e91918a4092fde7070bdf69eb 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -2293,7 +2293,7 @@ _docker_inspect() { ;; --type) if [ -z "$preselected_type" ] ; then - COMPREPLY=( $( compgen -W "container image network node service volume" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "container image network node plugin service volume" -- "$cur" ) ) return fi ;; @@ -2315,6 +2315,7 @@ _docker_inspect() { $(__docker_images) $(__docker_networks) $(__docker_nodes) + $(__docker_plugins_installed) $(__docker_services) $(__docker_volumes) " -- "$cur" ) ) @@ -2331,6 +2332,9 @@ _docker_inspect() { node) __docker_complete_nodes ;; + plugin) + __docker_complete_plugins_installed + ;; service) __docker_complete_services ;; From 09cd31b128ab746774791a02a9b6572feeaedde8 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 9 Dec 2016 10:02:44 -0800 Subject: [PATCH 46/68] Windows: Fix crash in docker system prune Signed-off-by: John Howard (cherry picked from commit e5900ee9bf0071c6a3ebaf813f82d5345c25d8f0) Signed-off-by: Victor Vieux --- pkg/term/windows/ansi_reader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/term/windows/ansi_reader.go b/pkg/term/windows/ansi_reader.go index 0bccf878f91352052e0c2be5816f2d60cd731d4d..cb0b88356df8591187f307b41d2073746ce21e1a 100644 --- a/pkg/term/windows/ansi_reader.go +++ b/pkg/term/windows/ansi_reader.go @@ -115,6 +115,8 @@ func readInputEvents(fd uintptr, maxBytes int) ([]winterm.INPUT_RECORD, error) { countRecords := maxBytes / recordSize if countRecords > ansiterm.MAX_INPUT_EVENTS { countRecords = ansiterm.MAX_INPUT_EVENTS + } else if countRecords == 0 { + countRecords = 1 } logger.Debugf("[windows] readInputEvents: Reading %v records (buffer size %v, record size %v)", countRecords, maxBytes, recordSize) From d8e435ab6cb51936f922fe4c9feac46cedd17e8f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 7 Dec 2016 17:35:09 -0800 Subject: [PATCH 47/68] remove old media type compat for plugins Signed-off-by: Victor Vieux (cherry picked from commit f644e758bd58f7b045a52b29038ae0043b0c9e3d) Signed-off-by: Victor Vieux --- distribution/pull_v2.go | 3 +-- docs/extend/config.md | 4 ++-- plugin/distribution/pull.go | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/distribution/pull_v2.go b/distribution/pull_v2.go index 5bfb328a18e4fc66a249d3e11f21491c09bac0f4..806ca85382fbc6b51f584bd8402b61d322b6570f 100644 --- a/distribution/pull_v2.go +++ b/distribution/pull_v2.go @@ -355,8 +355,7 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named) (tagUpdat } if m, ok := manifest.(*schema2.DeserializedManifest); ok { - if m.Manifest.Config.MediaType == schema2.MediaTypePluginConfig || - m.Manifest.Config.MediaType == "application/vnd.docker.plugin.image.v0+json" { //TODO: remove this v0 before 1.13 GA + if m.Manifest.Config.MediaType == schema2.MediaTypePluginConfig { return false, errMediaTypePlugin } } diff --git a/docs/extend/config.md b/docs/extend/config.md index 93ed4958a3e4b485c0f40942fcfbb31dc4533643..eca33803c2aff6663d446bfa01ce79492e9ec721 100644 --- a/docs/extend/config.md +++ b/docs/extend/config.md @@ -14,7 +14,7 @@ keywords: "API, Usage, plugins, documentation, developer" --> -# Plugin Config Version 0 of Plugin V2 +# Plugin Config Version 1 of Plugin V2 This document outlines the format of the V0 plugin configuration. The plugin config described herein was introduced in the Docker daemon in the [v1.12.0 @@ -25,7 +25,7 @@ configs can be serialized to JSON format with the following media types: Config Type | Media Type ------------- | ------------- -config | "application/vnd.docker.plugin.v0+json" +config | "application/vnd.docker.plugin.v1+json" ## *Config* Field Descriptions diff --git a/plugin/distribution/pull.go b/plugin/distribution/pull.go index 3e185cb0d31ef17f2f9712077faacabad6eefe7b..95743aa5770d87c3f8d4b6363155bb7a1e5b8355 100644 --- a/plugin/distribution/pull.go +++ b/plugin/distribution/pull.go @@ -153,8 +153,7 @@ func Pull(ref reference.Named, rs registry.Service, metaheader http.Header, auth logrus.Debugf("pull.go: error in json.Unmarshal(): %v", err) return nil, err } - if m.Config.MediaType != schema2.MediaTypePluginConfig && - m.Config.MediaType != "application/vnd.docker.plugin.image.v0+json" { //TODO: remove this v0 before 1.13 GA + if m.Config.MediaType != schema2.MediaTypePluginConfig { return nil, ErrUnsupportedMediaType } From 99eb943186befc2c3824345d860b7259d6d7250d Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Tue, 6 Dec 2016 18:52:47 -0800 Subject: [PATCH 48/68] service ps: Revert output to 1.12 behavior. - Display the ID column - Do not append the task ID in the name column - (NEW): Truncate task IDs, unless --no-trunc is specified Signed-off-by: Andrea Luzzardi (cherry picked from commit bbd2018ee19eff5594ae3986bf56fbcd0044699d) Signed-off-by: Victor Vieux --- cli/command/task/print.go | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/cli/command/task/print.go b/cli/command/task/print.go index 2995e9afb3814b49d1dd753db426971ae8d822c3..0f1c2cf724aa1ed618060483d7e7f7a080a0bb8d 100644 --- a/cli/command/task/print.go +++ b/cli/command/task/print.go @@ -14,11 +14,12 @@ import ( "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/cli/command" "github.com/docker/docker/cli/command/idresolver" + "github.com/docker/docker/pkg/stringid" "github.com/docker/go-units" ) const ( - psTaskItemFmt = "%s\t%s\t%s\t%s\t%s %s ago\t%s\t%s\n" + psTaskItemFmt = "%s\t%s\t%s\t%s\t%s\t%s %s ago\t%s\t%s\n" maxErrLength = 30 ) @@ -67,7 +68,7 @@ func Print(dockerCli *command.DockerCli, ctx context.Context, tasks []swarm.Task // Ignore flushing errors defer writer.Flush() - fmt.Fprintln(writer, strings.Join([]string{"NAME", "IMAGE", "NODE", "DESIRED STATE", "CURRENT STATE", "ERROR", "PORTS"}, "\t")) + fmt.Fprintln(writer, strings.Join([]string{"ID", "NAME", "IMAGE", "NODE", "DESIRED STATE", "CURRENT STATE", "ERROR", "PORTS"}, "\t")) if err := print(writer, ctx, tasks, resolver, noTrunc); err != nil { return err @@ -90,25 +91,36 @@ func PrintQuiet(dockerCli *command.DockerCli, tasks []swarm.Task) error { } func print(out io.Writer, ctx context.Context, tasks []swarm.Task, resolver *idresolver.IDResolver, noTrunc bool) error { - prevService := "" - prevSlot := 0 + prevName := "" for _, task := range tasks { - name, err := resolver.Resolve(ctx, task, task.ID) + id := task.ID + if !noTrunc { + id = stringid.TruncateID(id) + } + + serviceName, err := resolver.Resolve(ctx, swarm.Service{}, task.ServiceID) + if err != nil { + return err + } nodeValue, err := resolver.Resolve(ctx, swarm.Node{}, task.NodeID) if err != nil { return err } + name := "" + if task.Slot != 0 { + name = fmt.Sprintf("%v.%v", serviceName, task.Slot) + } else { + name = fmt.Sprintf("%v.%v", serviceName, task.NodeID) + } + // Indent the name if necessary indentedName := name - // Since the new format of the task name is .., we should only compare - // and here. - if prevService == task.ServiceID && prevSlot == task.Slot { + if name == prevName { indentedName = fmt.Sprintf(" \\_ %s", indentedName) } - prevService = task.ServiceID - prevSlot = task.Slot + prevName = name // Trim and quote the error message. taskErr := task.Status.Err @@ -134,6 +146,7 @@ func print(out io.Writer, ctx context.Context, tasks []swarm.Task, resolver *idr fmt.Fprintf( out, psTaskItemFmt, + id, indentedName, image, nodeValue, From 1f4dc1158c1baa81c046db36ffc87deafb72f175 Mon Sep 17 00:00:00 2001 From: Lajos Papp Date: Tue, 6 Dec 2016 12:15:10 +0100 Subject: [PATCH 49/68] Note about potential plugin issue when upgrading Fixes: #29172 Fixes: https://github.com/docker/for-mac/issues/1000 Signed-off-by: Lajos Papp Signed-off-by: lalyos (cherry picked from commit acd847cd569d35b63cb00446badf9d0ea83e214c) Signed-off-by: Victor Vieux --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f7a3d3e8ea61bb45b9afe4a9a8b8796ffddd83..1d10d5cf24d6c51dc0349dc40645319e5f127fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ be found. ## 1.13.0 (2016-12-08) +**IMPORTANT**: In Docker 1.13 managed plugin api changed from the experimental +version introduced in Docker 1.12. Plugins installed with Docker 1.12 should +therefore be **uninstalled** _before_ upgrading to Docker 1.13. You can +uninstall plugins using the `docker plugin rm` command. + +If you have already upgraded to Docker 1.13 without uninstalling plugins, and +get this error message during docker daemon startup; + + Error starting daemon: json: cannot unmarshal string into Go value of type types.PluginEnv + +Take the following steps to manually remove all plugins; + +- Remove plugins.json from: `/var/lib/docker/plugins/` +- Restart Docker +- Reinstall your plugins + ### Builder + Add capability to specify images used as a cache source on build. These images do not need to have local parent chain and can be pulled from other registries [#26839](https://github.com/docker/docker/pull/26839) + (experimental) Add option to squash image layers to the FROM image after successful builds [#22641](https://github.com/docker/docker/pull/22641) From 09296e053dc46304070a2ea76a41bd767bb1f643 Mon Sep 17 00:00:00 2001 From: Misty Stanley-Jones Date: Fri, 9 Dec 2016 14:41:51 -0800 Subject: [PATCH 50/68] Edits to plugin upgrade warning Signed-off-by: Misty Stanley-Jones (cherry picked from commit 66cbb5a552e21cc373e34771f6c97bb3e95f4fba) Signed-off-by: Victor Vieux --- CHANGELOG.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d10d5cf24d6c51dc0349dc40645319e5f127fc3..125bcc8970ef17db04c0af9598c9295e87eee2ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,21 +7,21 @@ be found. ## 1.13.0 (2016-12-08) -**IMPORTANT**: In Docker 1.13 managed plugin api changed from the experimental -version introduced in Docker 1.12. Plugins installed with Docker 1.12 should -therefore be **uninstalled** _before_ upgrading to Docker 1.13. You can -uninstall plugins using the `docker plugin rm` command. +**IMPORTANT**: In Docker 1.13, the managed plugin api changed, as compared to the experimental +version introduced in Docker 1.12. You must **uninstall** plugins which you installed with Docker 1.12 +_before_ upgrading to Docker 1.13. You can uninstall plugins using the `docker plugin rm` command. -If you have already upgraded to Docker 1.13 without uninstalling plugins, and -get this error message during docker daemon startup; +If you have already upgraded to Docker 1.13 without uninstalling +previously-installed plugins, you may see this message when the Docker daemon +starts: Error starting daemon: json: cannot unmarshal string into Go value of type types.PluginEnv -Take the following steps to manually remove all plugins; +To manually remove all plugins and resolve this problem, take the following steps: -- Remove plugins.json from: `/var/lib/docker/plugins/` -- Restart Docker -- Reinstall your plugins +1. Remove plugins.json from: `/var/lib/docker/plugins/`. +2. Restart Docker. Verify that the Docker daemon starts with no errors. +3. Reinstall your plugins. ### Builder + Add capability to specify images used as a cache source on build. These images do not need to have local parent chain and can be pulled from other registries [#26839](https://github.com/docker/docker/pull/26839) @@ -605,7 +605,7 @@ installing docker, please make sure to update them accordingly. ### DEPRECATION -* Environment variables `DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE` and `DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE` have been renamed +* Environment variables `DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE` and `DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE` have been renamed to `DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE` and `DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE` respectively [#22574](https://github.com/docker/docker/pull/22574) * Remove deprecated `syslog-tag`, `gelf-tag`, `fluentd-tag` log option in favor of the more generic `tag` one [#22620](https://github.com/docker/docker/pull/22620) * Remove deprecated feature of passing HostConfig at API container start [#22570](https://github.com/docker/docker/pull/22570) @@ -776,7 +776,7 @@ installing docker, please make sure to update them accordingly. - Fix a panic that could occur when cleanup after a container started with invalid parameters ([#21716](https://github.com/docker/docker/pull/21716)) - Fix a race with event timers stopping early ([#21692](https://github.com/docker/docker/pull/21692)) - Fix race conditions in the layer store, potentially corrupting the map and crashing the process ([#21677](https://github.com/docker/docker/pull/21677)) -- Un-deprecate auto-creation of host directories for mounts. This feature was marked deprecated in ([#21666](https://github.com/docker/docker/pull/21666)) +- Un-deprecate auto-creation of host directories for mounts. This feature was marked deprecated in ([#21666](https://github.com/docker/docker/pull/21666)) Docker 1.9, but was decided to be too much of a backward-incompatible change, so it was decided to keep the feature. + It is now possible for containers to share the NET and IPC namespaces when `userns` is enabled ([#21383](https://github.com/docker/docker/pull/21383)) + `docker inspect ` will now expose the rootfs layers ([#21370](https://github.com/docker/docker/pull/21370)) From 20a7e39728c8ef641c7ae2ebfddb4f6aa773a2e2 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 2 Dec 2016 08:11:30 -0800 Subject: [PATCH 51/68] Fix missing IPAM options in swarm network mode This fix tries to fix the issue raised in 29044 where the IPAM options is missing in swarm network mode after the service is deployed. Before the service is deployed, the IPAM options is available. The reason for the issue is that, before service is deployed, `network inspect` is querying the swarm and obtained the correct information. However, after service is deployed, swarm executor does not pass the IPAM options to the backend (daemon). Also after service is deployed, `network inspect` is actually querying the local daemon for information. At this time the network information with missing IPAM options is returned. This fix fixes the issue by updating the swarm network allocator and swarm executor. A separate PR for swarmkit will be opened. An integration test has been added to cover the change. This fix fixes 29044. Signed-off-by: Yong Tang (cherry picked from commit 4d958e99c178f7cd4196ed901c2834ae13f0f7d0) Signed-off-by: Victor Vieux --- .../cluster/executor/container/container.go | 3 ++- integration-cli/docker_cli_swarm_test.go | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/daemon/cluster/executor/container/container.go b/daemon/cluster/executor/container/container.go index d44ae2945b9d27bbeeb10e73a8be11c4453c0457..f033ad545ecf37cfbe18c844c0f7b702f2cf90ea 100644 --- a/daemon/cluster/executor/container/container.go +++ b/daemon/cluster/executor/container/container.go @@ -566,7 +566,8 @@ func (c *containerConfig) networkCreateRequest(name string) (clustertypes.Networ // ID: na.Network.ID, Driver: na.Network.DriverState.Name, IPAM: &network.IPAM{ - Driver: na.Network.IPAM.Driver.Name, + Driver: na.Network.IPAM.Driver.Name, + Options: na.Network.IPAM.Driver.Options, }, Options: na.Network.DriverState.Options, Labels: na.Network.Spec.Annotations.Labels, diff --git a/integration-cli/docker_cli_swarm_test.go b/integration-cli/docker_cli_swarm_test.go index b9a2ca7652d6dd926afb27d42fbf8b8d595d44c8..facb628574b975812b9ade85d51a334fec4d0ebc 100644 --- a/integration-cli/docker_cli_swarm_test.go +++ b/integration-cli/docker_cli_swarm_test.go @@ -1041,3 +1041,25 @@ Resources: c.Assert(err, checker.IsNil, check.Commentf(out)) c.Assert(out, checker.Contains, expectedOutput, check.Commentf(out)) } + +func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) { + d := s.AddDaemon(c, true, true) + + out, err := d.Cmd("network", "create", "-d", "overlay", "--ipam-opt", "foo=bar", "foo") + c.Assert(err, checker.IsNil, check.Commentf(out)) + c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") + + out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") + c.Assert(err, checker.IsNil, check.Commentf(out)) + c.Assert(strings.TrimSpace(out), checker.Equals, "map[foo:bar]") + + out, err = d.Cmd("service", "create", "--network=foo", "--name", "top", "busybox", "top") + c.Assert(err, checker.IsNil, check.Commentf(out)) + + // make sure task has been deployed. + waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) + + out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") + c.Assert(err, checker.IsNil, check.Commentf(out)) + c.Assert(strings.TrimSpace(out), checker.Equals, "map[foo:bar]") +} From 20d6f23b55229b422c505d7056fd517a9027cf6a Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Tue, 6 Dec 2016 00:10:08 +1100 Subject: [PATCH 52/68] apparmor: switch IsLoaded to return bool Signed-off-by: Aleksa Sarai (cherry picked from commit e440a57a793feb15c0f06d27178ee8241a2a9081) Signed-off-by: Victor Vieux --- daemon/apparmor_default.go | 2 +- profiles/apparmor/apparmor.go | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/daemon/apparmor_default.go b/daemon/apparmor_default.go index e4065b4ad9e8d9f4eaf8f12677488674fa1c7411..790e14b656995eef89d27efdb3b80f286200c5d4 100644 --- a/daemon/apparmor_default.go +++ b/daemon/apparmor_default.go @@ -21,7 +21,7 @@ func installDefaultAppArmorProfile() { // Allow daemon to run if loading failed, but are active // (possibly through another run, manually, or via system startup) for _, policy := range apparmorProfiles { - if err := aaprofile.IsLoaded(policy); err != nil { + if loaded, err := aaprofile.IsLoaded(policy); err != nil || !loaded { logrus.Errorf("AppArmor enabled on system but the %s profile could not be loaded.", policy) } } diff --git a/profiles/apparmor/apparmor.go b/profiles/apparmor/apparmor.go index 8859a41b405fbba026d81db9fec10fd154a33ef4..5132ebe00813fdabdf8d94de7f09e5e66a161e31 100644 --- a/profiles/apparmor/apparmor.go +++ b/profiles/apparmor/apparmor.go @@ -95,22 +95,28 @@ func InstallDefault(name string) error { return nil } -// IsLoaded checks if a passed profile has been loaded into the kernel. -func IsLoaded(name string) error { +// IsLoaded checks if a profile with the given name has been loaded into the +// kernel. +func IsLoaded(name string) (bool, error) { file, err := os.Open("/sys/kernel/security/apparmor/profiles") if err != nil { - return err + return false, err } defer file.Close() r := bufio.NewReader(file) for { p, err := r.ReadString('\n') + if err == io.EOF { + break + } if err != nil { - return err + return false, err } if strings.HasPrefix(p, name+" ") { - return nil + return true, nil } } + + return false, nil } From 80c3ed1c0cdf407b73f1c7bc0a0a52dda88702d1 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Tue, 6 Dec 2016 00:12:17 +1100 Subject: [PATCH 53/68] daemon: switch to 'ensure' workflow for AppArmor profiles In certain cases (unattended upgrades), system services can disable loaded AppArmor profiles. However, since /etc being read-only is a supported setup we cannot just write a copy of the profile to /etc/apparmor.d. Instead, dynamically load the docker-default AppArmor profile if a container is started with that profile set. This code will short-cut if the profile is already loaded. Fixes: 2f7596aaef3a ("apparmor: do not save profile to /etc/apparmor.d") Signed-off-by: Aleksa Sarai (cherry picked from commit 567ef8e7858ca4f282f598ba1f5a951cbad39e83) Signed-off-by: Victor Vieux --- daemon/apparmor_default.go | 28 ++++++++++++++---------- daemon/apparmor_default_unsupported.go | 3 ++- daemon/daemon.go | 5 ++++- daemon/oci_linux.go | 19 ++++++++++++++-- integration-cli/docker_cli_swarm_test.go | 2 +- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/daemon/apparmor_default.go b/daemon/apparmor_default.go index 790e14b656995eef89d27efdb3b80f286200c5d4..09dd0541b872cd0a592acb1cbc32a01f224744cd 100644 --- a/daemon/apparmor_default.go +++ b/daemon/apparmor_default.go @@ -3,7 +3,8 @@ package daemon import ( - "github.com/Sirupsen/logrus" + "fmt" + aaprofile "github.com/docker/docker/profiles/apparmor" "github.com/opencontainers/runc/libcontainer/apparmor" ) @@ -13,18 +14,23 @@ const ( defaultApparmorProfile = "docker-default" ) -func installDefaultAppArmorProfile() { +func ensureDefaultAppArmorProfile() error { if apparmor.IsEnabled() { + loaded, err := aaprofile.IsLoaded(defaultApparmorProfile) + if err != nil { + return fmt.Errorf("Could not check if %s AppArmor profile was loaded: %s", defaultApparmorProfile, err) + } + + // Nothing to do. + if loaded { + return nil + } + + // Load the profile. if err := aaprofile.InstallDefault(defaultApparmorProfile); err != nil { - apparmorProfiles := []string{defaultApparmorProfile} - - // Allow daemon to run if loading failed, but are active - // (possibly through another run, manually, or via system startup) - for _, policy := range apparmorProfiles { - if loaded, err := aaprofile.IsLoaded(policy); err != nil || !loaded { - logrus.Errorf("AppArmor enabled on system but the %s profile could not be loaded.", policy) - } - } + return fmt.Errorf("AppArmor enabled on system but the %s profile could not be loaded.", defaultApparmorProfile) } } + + return nil } diff --git a/daemon/apparmor_default_unsupported.go b/daemon/apparmor_default_unsupported.go index f186a68af92bcdd7387b0f3ba19e8c280345f997..cd2dd9702ef256acc70e63af31c085b6e81d6aaf 100644 --- a/daemon/apparmor_default_unsupported.go +++ b/daemon/apparmor_default_unsupported.go @@ -2,5 +2,6 @@ package daemon -func installDefaultAppArmorProfile() { +func ensureDefaultAppArmorProfile() error { + return nil } diff --git a/daemon/daemon.go b/daemon/daemon.go index 7c2ce505596b7b86542687f6abd8b46a395874a4..6a339a2870a2f68b6fd5b4df2d0f4d779fc81188 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -524,7 +524,10 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot logrus.Warnf("Failed to configure golang's threads limit: %v", err) } - installDefaultAppArmorProfile() + if err := ensureDefaultAppArmorProfile(); err != nil { + logrus.Errorf(err.Error()) + } + daemonRepo := filepath.Join(config.Root, "containers") if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) { return nil, err diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index bf21df86c490329dcfa7afb65fba50c65eba491a..1daefc587beed9dda01388d26526e0bcfcf6b505 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -733,12 +733,27 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } if apparmor.IsEnabled() { - appArmorProfile := "docker-default" - if len(c.AppArmorProfile) > 0 { + var appArmorProfile string + if c.AppArmorProfile != "" { appArmorProfile = c.AppArmorProfile } else if c.HostConfig.Privileged { appArmorProfile = "unconfined" + } else { + appArmorProfile = "docker-default" + } + + if appArmorProfile == "docker-default" { + // Unattended upgrades and other fun services can unload AppArmor + // profiles inadvertently. Since we cannot store our profile in + // /etc/apparmor.d, nor can we practically add other ways of + // telling the system to keep our profile loaded, in order to make + // sure that we keep the default profile enabled we dynamically + // reload it if necessary. + if err := ensureDefaultAppArmorProfile(); err != nil { + return nil, err + } } + s.Process.ApparmorProfile = appArmorProfile } s.Process.SelinuxLabel = c.GetProcessLabel() diff --git a/integration-cli/docker_cli_swarm_test.go b/integration-cli/docker_cli_swarm_test.go index facb628574b975812b9ade85d51a334fec4d0ebc..0dc112079e6ed4357702cdf35b6ee8eb8d60122e 100644 --- a/integration-cli/docker_cli_swarm_test.go +++ b/integration-cli/docker_cli_swarm_test.go @@ -1057,7 +1057,7 @@ func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // make sure task has been deployed. - waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) + waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") c.Assert(err, checker.IsNil, check.Commentf(out)) From 1a86cbb0312e33232a45cf9cd810a71457e65985 Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Mon, 12 Dec 2016 12:56:44 -0800 Subject: [PATCH 54/68] When plugin enable fails, unmount PropagatedMount. Signed-off-by: Anusha Ragunathan (cherry picked from commit cef443bddf2a185b3afa2f5c7333fd461c87ae74) Signed-off-by: Victor Vieux --- plugin/manager_linux.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go index 9764ec6a74aef5bbd387dfbc90394b2dc2cfb9c1..20ea0c3bcc5ca440cb09b3237149350ec8cd36d2 100644 --- a/plugin/manager_linux.go +++ b/plugin/manager_linux.go @@ -41,6 +41,11 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { } if err := pm.containerdClient.Create(p.GetID(), "", "", specs.Spec(*spec), attachToLog(p.GetID())); err != nil { + if p.PropagatedMount != "" { + if err := mount.Unmount(p.PropagatedMount); err != nil { + logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err) + } + } return err } From 4c65bf9a10b412660043014dec49535903775013 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 12 Dec 2016 15:46:36 -0500 Subject: [PATCH 55/68] [integration-cli] skip plugin tests on non-x86 Due to the test plugins being architecture specific, these tests fail to start the plugin (even though they don't fail yet) Temporary fix until we can build architecture specific test plugins. Signed-off-by: Christopher Jones (cherry picked from commit ebff8c79a3b834c555f92e673c604f14fa0afa33) Signed-off-by: Victor Vieux --- .../docker_cli_authz_plugin_v2_test.go | 6 +++-- .../docker_cli_daemon_plugins_test.go | 12 +++++----- integration-cli/docker_cli_events_test.go | 2 +- integration-cli/docker_cli_inspect_test.go | 2 +- .../docker_cli_network_unix_test.go | 2 +- integration-cli/docker_cli_plugins_test.go | 22 +++++++++---------- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/integration-cli/docker_cli_authz_plugin_v2_test.go b/integration-cli/docker_cli_authz_plugin_v2_test.go index f407d8bdcad260b9b223c62e420753dd5fc78e61..8a669fb3794734b1253b52de463dd5dfc6440bd1 100644 --- a/integration-cli/docker_cli_authz_plugin_v2_test.go +++ b/integration-cli/docker_cli_authz_plugin_v2_test.go @@ -41,7 +41,7 @@ func (s *DockerAuthzV2Suite) TearDownTest(c *check.C) { } func (s *DockerAuthzV2Suite) TestAuthZPluginAllowNonVolumeRequest(c *check.C) { - testRequires(c, IsAmd64) + testRequires(c, DaemonIsLinux, IsAmd64, Network) // Install authz plugin _, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", authzPluginNameWithTag) c.Assert(err, checker.IsNil) @@ -71,7 +71,7 @@ func (s *DockerAuthzV2Suite) TestAuthZPluginAllowNonVolumeRequest(c *check.C) { } func (s *DockerAuthzV2Suite) TestAuthZPluginRejectVolumeRequests(c *check.C) { - testRequires(c, IsAmd64) + testRequires(c, DaemonIsLinux, IsAmd64, Network) // Install authz plugin _, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", authzPluginNameWithTag) c.Assert(err, checker.IsNil) @@ -111,6 +111,7 @@ func (s *DockerAuthzV2Suite) TestAuthZPluginRejectVolumeRequests(c *check.C) { } func (s *DockerAuthzV2Suite) TestAuthZPluginBadManifestFailsDaemonStart(c *check.C) { + testRequires(c, DaemonIsLinux, IsAmd64, Network) // Install authz plugin with bad manifest _, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", authzPluginBadManifestName) c.Assert(err, checker.IsNil) @@ -123,6 +124,7 @@ func (s *DockerAuthzV2Suite) TestAuthZPluginBadManifestFailsDaemonStart(c *check } func (s *DockerAuthzV2Suite) TestNonexistentAuthZPluginFailsDaemonStart(c *check.C) { + testRequires(c, DaemonIsLinux, Network) // start the daemon with a non-existent authz plugin, it will error c.Assert(s.d.Restart("--authorization-plugin="+nonexistentAuthzPluginName), check.NotNil) diff --git a/integration-cli/docker_cli_daemon_plugins_test.go b/integration-cli/docker_cli_daemon_plugins_test.go index 7ee068b37abac48e939cc4ff401cfd18f8f759a0..6061a98c8a7be1fccd83f8d8281ea2bfcded7f95 100644 --- a/integration-cli/docker_cli_daemon_plugins_test.go +++ b/integration-cli/docker_cli_daemon_plugins_test.go @@ -16,7 +16,7 @@ import ( // TestDaemonRestartWithPluginEnabled tests state restore for an enabled plugin func (s *DockerDaemonSuite) TestDaemonRestartWithPluginEnabled(c *check.C) { - testRequires(c, Network) + testRequires(c, IsAmd64, Network) if err := s.d.Start(); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -49,7 +49,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithPluginEnabled(c *check.C) { // TestDaemonRestartWithPluginDisabled tests state restore for a disabled plugin func (s *DockerDaemonSuite) TestDaemonRestartWithPluginDisabled(c *check.C) { - testRequires(c, Network) + testRequires(c, IsAmd64, Network) if err := s.d.Start(); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -80,7 +80,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithPluginDisabled(c *check.C) { // TestDaemonKillLiveRestoreWithPlugins SIGKILLs daemon started with --live-restore. // Plugins should continue to run. func (s *DockerDaemonSuite) TestDaemonKillLiveRestoreWithPlugins(c *check.C) { - testRequires(c, Network, IsAmd64) + testRequires(c, IsAmd64, Network) if err := s.d.Start("--live-restore"); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -113,7 +113,7 @@ func (s *DockerDaemonSuite) TestDaemonKillLiveRestoreWithPlugins(c *check.C) { // TestDaemonShutdownLiveRestoreWithPlugins SIGTERMs daemon started with --live-restore. // Plugins should continue to run. func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) { - testRequires(c, Network, IsAmd64) + testRequires(c, IsAmd64, Network) if err := s.d.Start("--live-restore"); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -145,7 +145,7 @@ func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) // TestDaemonShutdownWithPlugins shuts down running plugins. func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) { - testRequires(c, Network) + testRequires(c, IsAmd64, Network) if err := s.d.Start(); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -184,7 +184,7 @@ func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) { // TestVolumePlugin tests volume creation using a plugin. func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) { - testRequires(c, Network, IsAmd64) + testRequires(c, IsAmd64, Network) volName := "plugin-volume" destDir := "/tmp/data/" diff --git a/integration-cli/docker_cli_events_test.go b/integration-cli/docker_cli_events_test.go index f7048dd3da08d682f28ce354f487b1e70cb873d9..1fbfc742de84a8e785610a90abb0ebd9d6c42c7a 100644 --- a/integration-cli/docker_cli_events_test.go +++ b/integration-cli/docker_cli_events_test.go @@ -276,7 +276,7 @@ func (s *DockerSuite) TestEventsImageLoad(c *check.C) { } func (s *DockerSuite) TestEventsPluginOps(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) since := daemonUnixTime(c) diff --git a/integration-cli/docker_cli_inspect_test.go b/integration-cli/docker_cli_inspect_test.go index 2f2a918f0f98622d55819b73874955b72a772e15..8935995c68a98bcb1262a3ba2cb30df0996cbb72 100644 --- a/integration-cli/docker_cli_inspect_test.go +++ b/integration-cli/docker_cli_inspect_test.go @@ -419,7 +419,7 @@ func (s *DockerSuite) TestInspectAmpersand(c *check.C) { } func (s *DockerSuite) TestInspectPlugin(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) _, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pNameWithTag) c.Assert(err, checker.IsNil) diff --git a/integration-cli/docker_cli_network_unix_test.go b/integration-cli/docker_cli_network_unix_test.go index 82fa97a55c2e06ea439df2759945f2cfd422016f..829ad95d6ad528e065fad5000527664e0238e842 100644 --- a/integration-cli/docker_cli_network_unix_test.go +++ b/integration-cli/docker_cli_network_unix_test.go @@ -769,7 +769,7 @@ func (s *DockerNetworkSuite) TestDockerNetworkDriverOptions(c *check.C) { } func (s *DockerNetworkSuite) TestDockerPluginV2NetworkDriver(c *check.C) { - testRequires(c, DaemonIsLinux, Network, IsAmd64) + testRequires(c, DaemonIsLinux, IsAmd64, Network) var ( npName = "tiborvass/test-docker-netplugin" diff --git a/integration-cli/docker_cli_plugins_test.go b/integration-cli/docker_cli_plugins_test.go index 7b9a69cb7504f10f9a548125f92604a42814ebcc..cfc02518e4c89d4a2cad7726df09a52ede273425 100644 --- a/integration-cli/docker_cli_plugins_test.go +++ b/integration-cli/docker_cli_plugins_test.go @@ -18,7 +18,7 @@ var ( ) func (s *DockerSuite) TestPluginBasicOps(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) _, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pNameWithTag) c.Assert(err, checker.IsNil) @@ -50,7 +50,7 @@ func (s *DockerSuite) TestPluginBasicOps(c *check.C) { } func (s *DockerSuite) TestPluginForceRemove(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pNameWithTag) c.Assert(err, checker.IsNil) @@ -63,7 +63,7 @@ func (s *DockerSuite) TestPluginForceRemove(c *check.C) { } func (s *DockerSuite) TestPluginActive(c *check.C) { - testRequires(c, DaemonIsLinux, Network, IsAmd64) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pNameWithTag) c.Assert(err, checker.IsNil) @@ -90,7 +90,7 @@ func (s *DockerSuite) TestPluginActive(c *check.C) { } func (s *DockerSuite) TestPluginInstallDisable(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", "--disable", pName) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Contains, pName) @@ -113,7 +113,7 @@ func (s *DockerSuite) TestPluginInstallDisable(c *check.C) { } func (s *DockerSuite) TestPluginInstallDisableVolumeLs(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", "--disable", pName) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Contains, pName) @@ -122,7 +122,7 @@ func (s *DockerSuite) TestPluginInstallDisableVolumeLs(c *check.C) { } func (s *DockerSuite) TestPluginSet(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _ := dockerCmd(c, "plugin", "install", "--grant-all-permissions", "--disable", pName) c.Assert(strings.TrimSpace(out), checker.Contains, pName) @@ -136,7 +136,7 @@ func (s *DockerSuite) TestPluginSet(c *check.C) { } func (s *DockerSuite) TestPluginInstallArgs(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _ := dockerCmd(c, "plugin", "install", "--grant-all-permissions", "--disable", pName, "DEBUG=1") c.Assert(strings.TrimSpace(out), checker.Contains, pName) @@ -145,14 +145,14 @@ func (s *DockerSuite) TestPluginInstallArgs(c *check.C) { } func (s *DockerSuite) TestPluginInstallImage(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "redis") c.Assert(err, checker.NotNil) c.Assert(out, checker.Contains, "content is not a plugin") } func (s *DockerSuite) TestPluginEnableDisableNegative(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pName) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Contains, pName) @@ -173,7 +173,7 @@ func (s *DockerSuite) TestPluginEnableDisableNegative(c *check.C) { } func (s *DockerSuite) TestPluginCreate(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) name := "foo/bar-driver" temp, err := ioutil.TempDir("", "foo") @@ -204,7 +204,7 @@ func (s *DockerSuite) TestPluginCreate(c *check.C) { } func (s *DockerSuite) TestPluginInspect(c *check.C) { - testRequires(c, DaemonIsLinux, Network) + testRequires(c, DaemonIsLinux, IsAmd64, Network) _, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", pNameWithTag) c.Assert(err, checker.IsNil) From f59aa96e3d7250a6efdae87669c8edb994e039bd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 15 Nov 2016 15:41:52 +0100 Subject: [PATCH 56/68] deprecate "top-level" network information in NetworkSettings When inspecting a container, `NetworkSettings` contains top-level information about the default ("bridge") network; `EndpointID`, `Gateway`, `GlobalIPv6Address`, `GlobalIPv6PrefixLen`, `IPAddress`, `IPPrefixLen`, `IPv6Gateway`, and `MacAddress`. These properties are deprecated in favor of per-network properties in `NetworkSettings.Networks`. These properties were already "deprecated" in docker 1.9, but kept around for backward compatibility. Refer to [#17538](https://github.com/docker/docker/pull/17538) for further information. This officially deprecates these properties, and marks them for removal in 1.16 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ae6f09b29ccbcd11c842d5b8cf319d7ce2da41be) Signed-off-by: Victor Vieux --- CHANGELOG.md | 1 + docs/deprecated.md | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125bcc8970ef17db04c0af9598c9295e87eee2ea..69631f4334fe7960a5875d7acc94cd2d43d9d750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,7 @@ To manually remove all plugins and resolve this problem, take the following step - Deprecate `MAINTAINER` in Dockerfile [#25466](https://github.com/docker/docker/pull/25466) - Deprecate `filter` param for endpoint `/images/json` [#27872](https://github.com/docker/docker/pull/27872) - Deprecate setting duplicate engine labels [#24533](https://github.com/docker/docker/pull/24533) +- Deprecate "top-level" network information in `NetworkSettings` [#28437](https://github.com/docker/docker/pull/28437) ## 1.12.3 (2016-10-26) diff --git a/docs/deprecated.md b/docs/deprecated.md index b8e6af1116411de55a3ba04b1b22228ec5417c9e..1298370ba93cdccab06e5f423320d2630a89b105 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -20,6 +20,26 @@ The following list of features are deprecated in Engine. To learn more about Docker Engine's deprecation policy, see [Feature Deprecation Policy](https://docs.docker.com/engine/#feature-deprecation-policy). + +### Top-level network properties in NetworkSettings + +**Deprecated In Release: v1.13.0** + +**Target For Removal In Release: v1.16** + +When inspecting a container, `NetworkSettings` contains top-level information +about the default ("bridge") network; + +`EndpointID`, `Gateway`, `GlobalIPv6Address`, `GlobalIPv6PrefixLen`, `IPAddress`, +`IPPrefixLen`, `IPv6Gateway`, and `MacAddress`. + +These properties are deprecated in favor of per-network properties in +`NetworkSettings.Networks`. These properties were already "deprecated" in +docker 1.9, but kept around for backward compatibility. + +Refer to [#17538](https://github.com/docker/docker/pull/17538) for further +information. + ## `filter` param for `/images/json` endpoint **Deprecated In Release: [v1.13.0](https://github.com/docker/docker/releases/tag/v1.13.0)** From 4aa72cb5a90d330aea406bc1fea182b8199b4486 Mon Sep 17 00:00:00 2001 From: allencloud Date: Thu, 8 Dec 2016 00:46:07 +0800 Subject: [PATCH 57/68] add missing status code 403 for services/create in docs Signed-off-by: allencloud (cherry picked from commit 19654fd71ec5b9b075863f4623fdb80a955f5187) Signed-off-by: Victor Vieux --- api/swagger.yaml | 4 ++++ docs/api/v1.24.md | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/api/swagger.yaml b/api/swagger.yaml index ffbeffb2ce91f25c2d207f20794cc00258e880ed..753dcb267e33b545641c75362be35f6582a6b02e 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -7138,6 +7138,10 @@ paths: example: ID: "ak7w3gjqoa3kuz8xcpnyy0pvl" Warning: "unable to pin image doesnotexist:latest to digest: image library/doesnotexist:latest not found" + 403: + description: "network is not eligible for services" + schema: + $ref: "#/definitions/ErrorResponse" 409: description: "name conflicts with an existing service" schema: diff --git a/docs/api/v1.24.md b/docs/api/v1.24.md index 81479c083e4d19fa9a2f0f3c2f953ef9f368d188..704ba8c0119cab548a095c9f708c96f7f4733791 100644 --- a/docs/api/v1.24.md +++ b/docs/api/v1.24.md @@ -4563,6 +4563,11 @@ image](#create-an-image) section for more details. ], "User": "33" }, + "Networks": [ + { + "Target": "overlay1" + } + ], "LogDriver": { "Name": "json-file", "Options": { @@ -4620,6 +4625,7 @@ image](#create-an-image) section for more details. **Status codes**: - **201** – no error +- **403** - network is not eligible for services - **406** – node is not part of a swarm - **409** – name conflicts with an existing object - **500** - server error From 54e52b97f4872284a4d22c6233f703f7e03eb8f6 Mon Sep 17 00:00:00 2001 From: yuexiao-wang Date: Fri, 9 Dec 2016 23:15:26 +0800 Subject: [PATCH 58/68] Update the option 'network' for docker build Signed-off-by: yuexiao-wang (cherry picked from commit cd317282c9b7d4dc68dce8df1d2f84e7a5654e72) Signed-off-by: Victor Vieux --- cli/command/image/build.go | 2 +- docs/reference/commandline/build.md | 5 ++--- man/docker-build.1.md | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cli/command/image/build.go b/cli/command/image/build.go index 1699b2c45c02a1369cc9db08b6f321be4532fa54..e3e7ff2b02f22338c176c155614bf8b05f1848e0 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -107,7 +107,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command { flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources") flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip") flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options") - flags.StringVar(&options.networkMode, "network", "default", "Connect a container to a network") + flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build") command.AddTrustedFlags(flags, true) diff --git a/docs/reference/commandline/build.md b/docs/reference/commandline/build.md index e96ebe5e1056d073fe4d319da47295d4dac3e24c..42c3ecf65fb636a564e2630f49108fa4321abe25 100644 --- a/docs/reference/commandline/build.md +++ b/docs/reference/commandline/build.md @@ -38,8 +38,7 @@ Options: --label value Set metadata for an image (default []) -m, --memory string Memory limit --memory-swap string Swap limit equal to memory plus swap: '-1' to enable unlimited swap - --network string Set the networking mode for the run commands - during build. + --network string Set the networking mode for the RUN instructions during build 'bridge': use default Docker bridge 'none': no networking 'container:': reuse another container's network stack @@ -54,7 +53,7 @@ Options: The format is ``. `number` must be greater than `0`. Unit is optional and can be `b` (bytes), `k` (kilobytes), `m` (megabytes), or `g` (gigabytes). If you omit the unit, the system uses bytes. - --squash Squash newly built layers into a single new layer (**Experimental Only**) + --squash Squash newly built layers into a single new layer (**Experimental Only**) -t, --tag value Name and optionally a tag in the 'name:tag' format (default []) --ulimit value Ulimit options (default []) ``` diff --git a/man/docker-build.1.md b/man/docker-build.1.md index aef3414879acb51acbeb78f3fe797fc84e897c9f..e41e378cb21fcb262b71ca5d1e4551079460af2e 100644 --- a/man/docker-build.1.md +++ b/man/docker-build.1.md @@ -2,7 +2,7 @@ % Docker Community % JUNE 2014 # NAME -docker-build - Build a new image from the source code at PATH +docker-build - Build an image from a Dockerfile # SYNOPSIS **docker build** @@ -130,7 +130,9 @@ set as the **URL**, the repository is cloned locally and then sent as the contex unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap. **--network**=*NETWORK* - + Set the networking mode for the RUN instructions during build. Supported standard + values are: `bridge`, `host`, `none` and `container:`. Any other value + is taken as a custom network's name or ID which this container should connect to. **--shm-size**=*SHM-SIZE* Size of `/dev/shm`. The format is ``. `number` must be greater than `0`. From 004ec96608e40acb01564c91c7676df55f7b1138 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 9 Dec 2016 14:21:45 -0800 Subject: [PATCH 59/68] Windows: Prompt fix Signed-off-by: John Howard (cherry picked from commit 30b8f084436a2a1d5e8523fcd2c5ea64cc805224) Signed-off-by: Victor Vieux --- cli/command/utils.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/command/utils.go b/cli/command/utils.go index 9f9a1ee80d5fe1569b645fa9dea1ddf6a6febb4e..1837ca41f0353f7c32f1e3f88b5c4989b4f75b83 100644 --- a/cli/command/utils.go +++ b/cli/command/utils.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" ) @@ -71,6 +72,11 @@ func PromptForConfirmation(ins *InStream, outs *OutStream, message string) bool fmt.Fprintf(outs, message) + // On Windows, force the use of the regular OS stdin stream. + if runtime.GOOS == "windows" { + ins = NewInStream(os.Stdin) + } + answer := "" n, _ := fmt.Fscan(ins, &answer) if n != 1 || (answer != "y" && answer != "Y") { From e65660da3d5351ec8c751985cd5e69fd2f917892 Mon Sep 17 00:00:00 2001 From: lixiaobing10051267 Date: Tue, 13 Dec 2016 18:06:18 +0800 Subject: [PATCH 60/68] stack_config.md not exist and delete it Signed-off-by: lixiaobing10051267 (cherry picked from commit afaff51a8b59630f880493679906acc3acc3005a) Signed-off-by: Victor Vieux --- docs/reference/commandline/stack_deploy.md | 1 - docs/reference/commandline/stack_ls.md | 3 +-- docs/reference/commandline/stack_ps.md | 3 +-- docs/reference/commandline/stack_rm.md | 5 ++--- docs/reference/commandline/stack_services.md | 5 ++--- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/reference/commandline/stack_deploy.md b/docs/reference/commandline/stack_deploy.md index 54af3e4a593d4effebf4a49cda236b326ac44719..28b7604105a473ca2303c81e283b47326afc0498 100644 --- a/docs/reference/commandline/stack_deploy.md +++ b/docs/reference/commandline/stack_deploy.md @@ -92,7 +92,6 @@ axqh55ipl40h vossibility_vossibility-collector replicated 1/1 icecrime/ ## Related information -* [stack config](stack_config.md) * [stack ls](stack_ls.md) * [stack ps](stack_ps.md) * [stack rm](stack_rm.md) diff --git a/docs/reference/commandline/stack_ls.md b/docs/reference/commandline/stack_ls.md index 4cf6d68d67af37d4a22c75c84f760ec4b1791582..05c721549278ce01f55589dfa23d000e9d33a877 100644 --- a/docs/reference/commandline/stack_ls.md +++ b/docs/reference/commandline/stack_ls.md @@ -41,8 +41,7 @@ myapp 2 ## Related information -* [stack config](stack_config.md) * [stack deploy](stack_deploy.md) -* [stack rm](stack_rm.md) * [stack ps](stack_ps.md) +* [stack rm](stack_rm.md) * [stack services](stack_services.md) diff --git a/docs/reference/commandline/stack_ps.md b/docs/reference/commandline/stack_ps.md index 951c694e4ef8418347177102b1a0c12af2e85885..75223e07d9a908653a0e27004321250c5025c18b 100644 --- a/docs/reference/commandline/stack_ps.md +++ b/docs/reference/commandline/stack_ps.md @@ -45,8 +45,7 @@ The currently supported filters are: ## Related information -* [stack config](stack_config.md) * [stack deploy](stack_deploy.md) +* [stack rm](stack_ls.md) * [stack rm](stack_rm.md) * [stack services](stack_services.md) -* [stack ls](stack_ls.md) diff --git a/docs/reference/commandline/stack_rm.md b/docs/reference/commandline/stack_rm.md index a74b3ac817187674ec0a6eab982e09f4845de1b9..fd639978ecc88b9086399eedaa1db220eb3bf478 100644 --- a/docs/reference/commandline/stack_rm.md +++ b/docs/reference/commandline/stack_rm.md @@ -32,8 +32,7 @@ a manager node. ## Related information -* [stack config](stack_config.md) * [stack deploy](stack_deploy.md) -* [stack services](stack_services.md) -* [stack ps](stack_ps.md) * [stack ls](stack_ls.md) +* [stack ps](stack_ps.md) +* [stack services](stack_services.md) diff --git a/docs/reference/commandline/stack_services.md b/docs/reference/commandline/stack_services.md index 0d809fb370c4fb2830c6db68ceec74a509d8579a..62779b4aa18cdf53888913e6bcec0c63fffe35ed 100644 --- a/docs/reference/commandline/stack_services.md +++ b/docs/reference/commandline/stack_services.md @@ -64,8 +64,7 @@ The currently supported filters are: ## Related information -* [stack config](stack_config.md) * [stack deploy](stack_deploy.md) -* [stack rm](stack_rm.md) -* [stack ps](stack_ps.md) * [stack ls](stack_ls.md) +* [stack ps](stack_ps.md) +* [stack rm](stack_rm.md) From 1dd2e2f545a2fc4e5fcba91b761760eef1936e1c Mon Sep 17 00:00:00 2001 From: yuexiao-wang Date: Wed, 14 Dec 2016 03:58:02 +0800 Subject: [PATCH 61/68] Fix the incorrect option name Signed-off-by: yuexiao-wang (cherry picked from commit b101e451f115840a05a6a6b412e06137f16fc64c) Signed-off-by: Victor Vieux --- man/docker-build.1.md | 2 +- man/docker-create.1.md | 8 ++++---- man/docker-network-connect.1.md | 4 ++-- man/docker-network-create.1.md | 4 ++-- man/docker-run.1.md | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/man/docker-build.1.md b/man/docker-build.1.md index e41e378cb21fcb262b71ca5d1e4551079460af2e..4beee88e4a0818f9525a3b3e4a8b7ea4ff2a0892 100644 --- a/man/docker-build.1.md +++ b/man/docker-build.1.md @@ -129,7 +129,7 @@ set as the **URL**, the repository is cloned locally and then sent as the contex `k` (kilobytes), `m` (megabytes), or `g` (gigabytes). If you don't specify a unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap. -**--network**=*NETWORK* +**--network**=*bridge* Set the networking mode for the RUN instructions during build. Supported standard values are: `bridge`, `host`, `none` and `container:`. Any other value is taken as a custom network's name or ID which this container should connect to. diff --git a/man/docker-create.1.md b/man/docker-create.1.md index a819904efa71c29a459b6374c3c3a2240641778b..3f8a076374c7d26166ac4940157f8d74b771aa7f 100644 --- a/man/docker-create.1.md +++ b/man/docker-create.1.md @@ -222,12 +222,12 @@ two memory nodes. **--ip**="" Sets the container's interface IPv4 address (e.g. 172.23.0.9) - It can only be used in conjunction with **--net** for user-defined networks + It can only be used in conjunction with **--network** for user-defined networks **--ip6**="" Sets the container's interface IPv6 address (e.g. 2001:db8::1b99) - It can only be used in conjunction with **--net** for user-defined networks + It can only be used in conjunction with **--network** for user-defined networks **--ipc**="" Default is to create a private IPC namespace (POSIX SysV IPC) for the container @@ -305,7 +305,7 @@ unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap. **--name**="" Assign a name to the container -**--net**="*bridge*" +**--network**="*bridge*" Set the Network mode for the container 'bridge': create a network stack on the default Docker bridge 'none': no networking @@ -404,7 +404,7 @@ unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap. Network Namespace - current sysctls allowed: Sysctls beginning with net.* - Note: if you use --net=host using these sysctls will not be allowed. + Note: if you use --network=host using these sysctls will not be allowed. **-t**, **--tty**=*true*|*false* Allocate a pseudo-TTY. The default is *false*. diff --git a/man/docker-network-connect.1.md b/man/docker-network-connect.1.md index 9cc012ea4f7b728f6b58fe459bcc409b60f5c15b..096ec77a4df764d7231d228dae4f6d4d60327e6b 100644 --- a/man/docker-network-connect.1.md +++ b/man/docker-network-connect.1.md @@ -19,10 +19,10 @@ the same network. $ docker network connect multi-host-network container1 ``` -You can also use the `docker run --net=` option to start a container and immediately connect it to a network. +You can also use the `docker run --network=` option to start a container and immediately connect it to a network. ```bash -$ docker run -itd --net=multi-host-network --ip 172.20.88.22 --ip6 2001:db8::8822 busybox +$ docker run -itd --network=multi-host-network --ip 172.20.88.22 --ip6 2001:db8::8822 busybox ``` You can pause, restart, and stop containers that are connected to a network. A container connects to its configured networks when it runs. diff --git a/man/docker-network-create.1.md b/man/docker-network-create.1.md index 3000bb213537d1970f7070bbb49892e35e4d6ebb..bea6fb44e4dc612319aa1db47ac14bec80b2feae 100644 --- a/man/docker-network-create.1.md +++ b/man/docker-network-create.1.md @@ -73,11 +73,11 @@ name conflicts. ## Connect containers -When you start a container use the `--net` flag to connect it to a network. +When you start a container use the `--network` flag to connect it to a network. This adds the `busybox` container to the `mynet` network. ```bash -$ docker run -itd --net=mynet busybox +$ docker run -itd --network=mynet busybox ``` If you want to add a container to a network after the container is already diff --git a/man/docker-run.1.md b/man/docker-run.1.md index a0cbd2ba630d3f3cf4a522cefe75bc23d1fc4673..0022e4d045b30ea653bc89cfd701c5654a3e16e3 100644 --- a/man/docker-run.1.md +++ b/man/docker-run.1.md @@ -319,12 +319,12 @@ redirection on the host system. **--ip**="" Sets the container's interface IPv4 address (e.g. 172.23.0.9) - It can only be used in conjunction with **--net** for user-defined networks + It can only be used in conjunction with **--network** for user-defined networks **--ip6**="" Sets the container's interface IPv6 address (e.g. 2001:db8::1b99) - It can only be used in conjunction with **--net** for user-defined networks + It can only be used in conjunction with **--network** for user-defined networks **--ipc**="" Default is to create a private IPC namespace (POSIX SysV IPC) for the container @@ -557,7 +557,7 @@ incompatible with any restart policy other than `none`. Network Namespace - current sysctls allowed: Sysctls beginning with net.* - If you use the `--net=host` option these sysctls will not be allowed. + If you use the `--network=host` option these sysctls will not be allowed. **--sig-proxy**=*true*|*false* Proxy received signals to the process (non-TTY mode only). SIGCHLD, SIGSTOP, and SIGKILL are not proxied. The default is *true*. From 47b838db40c9b75262b41e46c7bd682a9b590db5 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 13 Dec 2016 13:30:58 +0100 Subject: [PATCH 62/68] Update reference docs for service ps commit bbd2018ee19eff5594ae3986bf56fbcd0044699d changed the output format of `docker service ps`. this patch updates the reference docs to match the updated output format. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 5902fa33840507706e6e9b189fec0adeb20affa5) Signed-off-by: Victor Vieux --- docs/reference/commandline/service_create.md | 6 +- docs/reference/commandline/service_ps.md | 88 ++++++++++++-------- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/docs/reference/commandline/service_create.md b/docs/reference/commandline/service_create.md index 64d595ac7ae51b89df0b86e5f53439105a1465ad..76f11c0a7137fe00e60b3b418e686ddedce96f9b 100644 --- a/docs/reference/commandline/service_create.md +++ b/docs/reference/commandline/service_create.md @@ -534,9 +534,11 @@ service's name and the node's ID where it sits. ```bash $ docker service create --name hosttempl --hostname={% raw %}"{{.Node.ID}}-{{.Service.Name}}"{% endraw %} busybox top va8ew30grofhjoychbr6iot8c + $ docker service ps va8ew30grofhjoychbr6iot8c -NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -hosttempl.1.wo41w8hg8qan busybox:latest@sha256:29f5d56d12684887bdfa50dcd29fc31eea4aaf4ad3bec43daf19026a7ce69912 2e7a8a9c4da2 Running Running about a minute ago +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +wo41w8hg8qan hosttempl.1 busybox:latest@sha256:29f5d56d12684887bdfa50dcd29fc31eea4aaf4ad3bec43daf19026a7ce69912 2e7a8a9c4da2 Running Running about a minute ago + $ docker inspect --format={% raw %}"{{.Config.Hostname}}"{% endraw %} hosttempl.1.wo41w8hg8qanxwjwsg4kxpprj x3ti0erg11rjpg64m75kej2mz-hosttempl ``` diff --git a/docs/reference/commandline/service_ps.md b/docs/reference/commandline/service_ps.md index b824f53dab64770f63929cd72b7a25718bd9a311..830504ac17c978e61b8515ce7e2a63e1dcc966a6 100644 --- a/docs/reference/commandline/service_ps.md +++ b/docs/reference/commandline/service_ps.md @@ -40,36 +40,57 @@ The following command shows all the tasks that are part of the `redis` service: ```bash $ docker service ps redis -NAME IMAGE NODE DESIRED STATE CURRENT STATE -redis.1.0qihejybwf1x5vqi8lgzlgnpq redis:3.0.6 manager1 Running Running 8 seconds -redis.2.bk658fpbex0d57cqcwoe3jthu redis:3.0.6 worker2 Running Running 9 seconds -redis.3.5ls5s5fldaqg37s9pwayjecrf redis:3.0.6 worker1 Running Running 9 seconds -redis.4.8ryt076polmclyihzx67zsssj redis:3.0.6 worker1 Running Running 9 seconds -redis.5.1x0v8yomsncd6sbvfn0ph6ogc redis:3.0.6 manager1 Running Running 8 seconds -redis.6.71v7je3el7rrw0osfywzs0lko redis:3.0.6 worker2 Running Running 9 seconds -redis.7.4l3zm9b7tfr7cedaik8roxq6r redis:3.0.6 worker2 Running Running 9 seconds -redis.8.9tfpyixiy2i74ad9uqmzp1q6o redis:3.0.6 worker1 Running Running 9 seconds -redis.9.3w1wu13yuplna8ri3fx47iwad redis:3.0.6 manager1 Running Running 8 seconds -redis.10.8eaxrb2fqpbnv9x30vr06i6vt redis:3.0.6 manager1 Running Running 8 seconds + +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +0qihejybwf1x redis.1 redis:3.0.5 manager1 Running Running 8 seconds +bk658fpbex0d redis.2 redis:3.0.5 worker2 Running Running 9 seconds +5ls5s5fldaqg redis.3 redis:3.0.5 worker1 Running Running 9 seconds +8ryt076polmc redis.4 redis:3.0.5 worker1 Running Running 9 seconds +1x0v8yomsncd redis.5 redis:3.0.5 manager1 Running Running 8 seconds +71v7je3el7rr redis.6 redis:3.0.5 worker2 Running Running 9 seconds +4l3zm9b7tfr7 redis.7 redis:3.0.5 worker2 Running Running 9 seconds +9tfpyixiy2i7 redis.8 redis:3.0.5 worker1 Running Running 9 seconds +3w1wu13yupln redis.9 redis:3.0.5 manager1 Running Running 8 seconds +8eaxrb2fqpbn redis.10 redis:3.0.5 manager1 Running Running 8 seconds +``` + +In addition to _running_ tasks, the output also shows the task history. For +example, after updating the service to use the `redis:3.0.6` image, the output +may look like this: + +```bash +$ docker service ps redis + +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +50qe8lfnxaxk redis.1 redis:3.0.6 manager1 Running Running 6 seconds ago +ky2re9oz86r9 \_ redis.1 redis:3.0.5 manager1 Shutdown Shutdown 8 seconds ago +3s46te2nzl4i redis.2 redis:3.0.6 worker2 Running Running less than a second ago +nvjljf7rmor4 \_ redis.2 redis:3.0.6 worker2 Shutdown Rejected 23 seconds ago "No such image: redis@sha256:6…" +vtiuz2fpc0yb \_ redis.2 redis:3.0.5 worker2 Shutdown Shutdown 1 second ago +jnarweeha8x4 redis.3 redis:3.0.6 worker1 Running Running 3 seconds ago +vs448yca2nz4 \_ redis.3 redis:3.0.5 worker1 Shutdown Shutdown 4 seconds ago +jf1i992619ir redis.4 redis:3.0.6 worker1 Running Running 10 seconds ago +blkttv7zs8ee \_ redis.4 redis:3.0.5 worker1 Shutdown Shutdown 11 seconds ago ``` +The number of items in the task history is determined by the +`--task-history-limit` option that was set when initializing the swarm. You can +change the task history retention limit using the +[`docker swarm update`](swarm_update.md) command. + When deploying a service, docker resolves the digest for the service's image, and pins the service to that digest. The digest is not shown by -default, but is printed if `--no-trunc` is used; +default, but is printed if `--no-trunc` is used. The `--no-trunc` option +also shows the non-truncated task ID, and error-messages, as can be seen below; ```bash $ docker service ps --no-trunc redis -NAME IMAGE NODE DESIRED STATE CURRENT STATE -redis.1.0qihejybwf1x5vqi8lgzlgnpq redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 manager1 Running Running 28 seconds -redis.2.bk658fpbex0d57cqcwoe3jthu redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker2 Running Running 29 seconds -redis.3.5ls5s5fldaqg37s9pwayjecrf redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker1 Running Running 29 seconds -redis.4.8ryt076polmclyihzx67zsssj redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker1 Running Running 29 seconds -redis.5.1x0v8yomsncd6sbvfn0ph6ogc redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 manager1 Running Running 28 seconds -redis.6.71v7je3el7rrw0osfywzs0lko redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker2 Running Running 29 seconds -redis.7.4l3zm9b7tfr7cedaik8roxq6r redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker2 Running Running 29 seconds -redis.8.9tfpyixiy2i74ad9uqmzp1q6o redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker1 Running Running 29 seconds -redis.9.3w1wu13yuplna8ri3fx47iwad redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 manager1 Running Running 28 seconds -redis.10.8eaxrb2fqpbnv9x30vr06i6vt redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 manager1 Running Running 28 seconds + +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +50qe8lfnxaxksi9w2a704wkp7 redis.1 redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 manager1 Running Running 5 minutes ago +ky2re9oz86r9556i2szb8a8af \_ redis.1 redis:3.0.5@sha256:f8829e00d95672c48c60f468329d6693c4bdd28d1f057e755f8ba8b40008682e worker2 Shutdown Shutdown 5 minutes ago +bk658fpbex0d57cqcwoe3jthu redis.2 redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker2 Running Running 5 seconds +nvjljf7rmor4htv7l8rwcx7i7 \_ redis.2 redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842 worker2 Shutdown Rejected 5 minutes ago "No such image: redis@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842" ``` ## Filtering @@ -93,9 +114,10 @@ The `id` filter matches on all or a prefix of a task's ID. ```bash $ docker service ps -f "id=8" redis -NAME IMAGE NODE DESIRED STATE CURRENT STATE -redis.4.8ryt076polmclyihzx67zsssj redis:3.0.6 worker1 Running Running 9 seconds -redis.10.8eaxrb2fqpbnv9x30vr06i6vt redis:3.0.6 manager1 Running Running 8 seconds + +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +8ryt076polmc redis.4 redis:3.0.6 worker1 Running Running 9 seconds +8eaxrb2fqpbn redis.10 redis:3.0.6 manager1 Running Running 8 seconds ``` #### Name @@ -104,8 +126,8 @@ The `name` filter matches on task names. ```bash $ docker service ps -f "name=redis.1" redis -NAME IMAGE NODE DESIRED STATE CURRENT STATE -redis.1.0qihejybwf1x5vqi8lgzlgnpq redis:3.0.6 manager1 Running Running 8 seconds +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +qihejybwf1x5 redis.1 redis:3.0.6 manager1 Running Running 8 seconds ``` @@ -115,11 +137,11 @@ The `node` filter matches on a node name or a node ID. ```bash $ docker service ps -f "node=manager1" redis -NAME IMAGE NODE DESIRED STATE CURRENT STATE -redis.1.0qihejybwf1x5vqi8lgzlgnpq redis:3.0.6 manager1 Running Running 8 seconds -redis.5.1x0v8yomsncd6sbvfn0ph6ogc redis:3.0.6 manager1 Running Running 8 seconds -redis.9.3w1wu13yuplna8ri3fx47iwad redis:3.0.6 manager1 Running Running 8 seconds -redis.10.8eaxrb2fqpbnv9x30vr06i6vt redis:3.0.6 manager1 Running Running 8 seconds +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +0qihejybwf1x redis.1 redis:3.0.6 manager1 Running Running 8 seconds +1x0v8yomsncd redis.5 redis:3.0.6 manager1 Running Running 8 seconds +3w1wu13yupln redis.9 redis:3.0.6 manager1 Running Running 8 seconds +8eaxrb2fqpbn redis.10 redis:3.0.6 manager1 Running Running 8 seconds ``` From 4e08d8ef03426a63ea70dd83332f9c63b4145149 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 9 Dec 2016 12:53:10 -0500 Subject: [PATCH 63/68] Fix issues with plugin and `--live-restore` Fixes an issue when starting the daemon with live-restore where previously it was not set, plugins are not running. Fixes an issue when starting the daemon with live-restore, the plugin client (for interacting with the plugins HTTP interface) is not set, causing a panic when the plugin is called. Signed-off-by: Brian Goff (cherry picked from commit cb6633175c0de0a7ae155c4d378cd2379681554b) Signed-off-by: Victor Vieux --- .../docker_cli_daemon_plugins_test.go | 7 ++++- plugin/manager_linux.go | 26 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/integration-cli/docker_cli_daemon_plugins_test.go b/integration-cli/docker_cli_daemon_plugins_test.go index 6061a98c8a7be1fccd83f8d8281ea2bfcded7f95..961a6fb07824d1c233ca40cc730fa83c507cce93 100644 --- a/integration-cli/docker_cli_daemon_plugins_test.go +++ b/integration-cli/docker_cli_daemon_plugins_test.go @@ -145,7 +145,7 @@ func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) // TestDaemonShutdownWithPlugins shuts down running plugins. func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) { - testRequires(c, IsAmd64, Network) + testRequires(c, IsAmd64, Network, SameHostDaemon) if err := s.d.Start(); err != nil { c.Fatalf("Could not start daemon: %v", err) @@ -180,6 +180,11 @@ func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) { if out, ec, err := runCommandWithOutput(cmd); ec != 1 { c.Fatalf("Expected exit code '1', got %d err: %v output: %s ", ec, err, out) } + + s.d.Start("--live-restore") + cmd = exec.Command("pgrep", "-f", pluginProcessName) + out, _, err := runCommandWithOutput(cmd) + c.Assert(err, checker.IsNil, check.Commentf(out)) } // TestVolumePlugin tests volume creation using a plugin. diff --git a/plugin/manager_linux.go b/plugin/manager_linux.go index 20ea0c3bcc5ca440cb09b3237149350ec8cd36d2..340ea5a7c15eaa7be910d45a38929f55ff6b70f0 100644 --- a/plugin/manager_linux.go +++ b/plugin/manager_linux.go @@ -49,6 +49,10 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { return err } + return pm.pluginPostStart(p, c) +} + +func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error { client, err := plugins.NewClientWithTimeout("unix://"+filepath.Join(p.GetRuntimeSourcePath(), p.GetSocket()), nil, c.timeoutInSecs) if err != nil { c.restart = false @@ -59,12 +63,30 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error { p.SetPClient(client) pm.pluginStore.SetState(p, true) pm.pluginStore.CallHandler(p) - return nil } func (pm *Manager) restore(p *v2.Plugin) error { - return pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())) + if err := pm.containerdClient.Restore(p.GetID(), attachToLog(p.GetID())); err != nil { + return err + } + + if pm.liveRestore { + c := &controller{} + if pids, _ := pm.containerdClient.GetPidsForContainer(p.GetID()); len(pids) == 0 { + // plugin is not running, so follow normal startup procedure + return pm.enable(p, c, true) + } + + c.exitChan = make(chan bool) + c.restart = true + pm.mu.Lock() + pm.cMap[p] = c + pm.mu.Unlock() + return pm.pluginPostStart(p, c) + } + + return nil } func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd.Client) { From 39c9f1c02490d0d6ac5286eaeea39958287601cb Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Tue, 29 Nov 2016 14:05:12 -0800 Subject: [PATCH 64/68] Run overlay attachable test on a single daemon Signed-off-by: Dong Chen (cherry picked from commit e42d1bb4b229b825a388490f24885f06a7afc401) Signed-off-by: Victor Vieux --- integration-cli/docker_cli_swarm_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/integration-cli/docker_cli_swarm_test.go b/integration-cli/docker_cli_swarm_test.go index 0dc112079e6ed4357702cdf35b6ee8eb8d60122e..634fb955a5d74a0e9007df7476ade109f44ae7ec 100644 --- a/integration-cli/docker_cli_swarm_test.go +++ b/integration-cli/docker_cli_swarm_test.go @@ -344,29 +344,23 @@ func (s *DockerSwarmSuite) TestSwarmContainerAttachByNetworkId(c *check.C) { } func (s *DockerSwarmSuite) TestOverlayAttachable(c *check.C) { - d1 := s.AddDaemon(c, true, true) - d2 := s.AddDaemon(c, true, false) + d := s.AddDaemon(c, true, true) - out, err := d1.Cmd("network", "create", "-d", "overlay", "--attachable", "ovnet") + out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "ovnet") c.Assert(err, checker.IsNil, check.Commentf(out)) // validate attachable - out, err = d1.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") + out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") c.Assert(err, checker.IsNil, check.Commentf(out)) c.Assert(strings.TrimSpace(out), checker.Equals, "true") // validate containers can attache to this overlay network - out, err = d1.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "busybox", "top") - c.Assert(err, checker.IsNil, check.Commentf(out)) - out, err = d2.Cmd("run", "-d", "--network", "ovnet", "--name", "c2", "busybox", "top") + out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "busybox", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) // redo validation, there was a bug that the value of attachable changes after // containers attach to the network - out, err = d1.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") - c.Assert(err, checker.IsNil, check.Commentf(out)) - c.Assert(strings.TrimSpace(out), checker.Equals, "true") - out, err = d2.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") + out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") c.Assert(err, checker.IsNil, check.Commentf(out)) c.Assert(strings.TrimSpace(out), checker.Equals, "true") } From 8b31f217ff682b99bed79ff3fea7fb52a7f1a9a2 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 30 Nov 2016 16:02:45 -0500 Subject: [PATCH 65/68] Refcount graphdriver plugins properly Adds 2 new methods to v2 plugin `Acquire` and `Release` which allow refcounting directly at the plugin level instead of just the store. Since a graphdriver is initialized exactly once, and is really managed by a separate object, it didn't really seem right to call `getter.Get()` to refcount graphdriver plugins. On shutdown it was particularly weird where we'd either need to keep a driver reference in daemon, or keep a reference to the pluggin getter in the layer store, and even then still store extra details on if the graphdriver is a plugin or not. Instead the plugin proxy itself will handle calling the neccessary refcounting methods directly on the plugin object. Also adds a new interface in `plugingetter` to account for these new functions which are not going to be implemented by v1 plugins. Changes terms `plugingetter.CREATE` and `plugingetter.REMOVE` to `ACQUIRE` and `RELEASE` respectively, which seems to be better adjectives for what we're doing. Signed-off-by: Brian Goff (cherry picked from commit f29bbd16f5d2bb82d815ea59f8ef85fe59384c89) Signed-off-by: Victor Vieux --- daemon/graphdriver/plugin.go | 6 +++--- daemon/graphdriver/proxy.go | 15 +++++++++++++++ pkg/plugingetter/getter.go | 15 +++++++++++---- plugin/v2/plugin.go | 14 ++++++++++++++ volume/drivers/extpoint.go | 4 ++-- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/daemon/graphdriver/plugin.go b/daemon/graphdriver/plugin.go index 5e519ad255ade6c27cf0a1982bb93a4906b62c59..6a79c29681fe5525795f5c1ce925242ac580508c 100644 --- a/daemon/graphdriver/plugin.go +++ b/daemon/graphdriver/plugin.go @@ -22,10 +22,10 @@ func lookupPlugin(name, home string, opts []string, pg plugingetter.PluginGetter if err != nil { return nil, fmt.Errorf("Error looking up graphdriver plugin %s: %v", name, err) } - return newPluginDriver(name, home, opts, pl.Client()) + return newPluginDriver(name, home, opts, pl) } -func newPluginDriver(name, home string, opts []string, c pluginClient) (Driver, error) { - proxy := &graphDriverProxy{name, c} +func newPluginDriver(name, home string, opts []string, pl plugingetter.CompatPlugin) (Driver, error) { + proxy := &graphDriverProxy{name, pl.Client(), pl} return proxy, proxy.Init(filepath.Join(home, name), opts) } diff --git a/daemon/graphdriver/proxy.go b/daemon/graphdriver/proxy.go index 45aeba17371f5f40f9616efaa410c7e8133df3a9..713124b442ff770dec0a9e239b649d574e819671 100644 --- a/daemon/graphdriver/proxy.go +++ b/daemon/graphdriver/proxy.go @@ -6,11 +6,13 @@ import ( "io" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/plugingetter" ) type graphDriverProxy struct { name string client pluginClient + p plugingetter.CompatPlugin } type graphDriverRequest struct { @@ -35,6 +37,12 @@ type graphDriverInitRequest struct { } func (d *graphDriverProxy) Init(home string, opts []string) error { + if !d.p.IsV1() { + if cp, ok := d.p.(plugingetter.CountedPlugin); ok { + // always acquire here, it will be cleaned up on daemon shutdown + cp.Acquire() + } + } args := &graphDriverInitRequest{ Home: home, Opts: opts, @@ -167,6 +175,13 @@ func (d *graphDriverProxy) GetMetadata(id string) (map[string]string, error) { } func (d *graphDriverProxy) Cleanup() error { + if !d.p.IsV1() { + if cp, ok := d.p.(plugingetter.CountedPlugin); ok { + // always release + defer cp.Release() + } + } + args := &graphDriverRequest{} var ret graphDriverResponse if err := d.client.Call("GraphDriver.Cleanup", args, &ret); err != nil { diff --git a/pkg/plugingetter/getter.go b/pkg/plugingetter/getter.go index c568184b3ce47494b384b0d3b64a39e36f6787ca..b8096b96b22d16401977cf1fd13f3e503d5e6d3e 100644 --- a/pkg/plugingetter/getter.go +++ b/pkg/plugingetter/getter.go @@ -5,10 +5,10 @@ import "github.com/docker/docker/pkg/plugins" const ( // LOOKUP doesn't update RefCount LOOKUP = 0 - // CREATE increments RefCount - CREATE = 1 - // REMOVE decrements RefCount - REMOVE = -1 + // ACQUIRE increments RefCount + ACQUIRE = 1 + // RELEASE decrements RefCount + RELEASE = -1 ) // CompatPlugin is a abstraction to handle both v2(new) and v1(legacy) plugins. @@ -19,6 +19,13 @@ type CompatPlugin interface { IsV1() bool } +// CountedPlugin is a plugin which is reference counted. +type CountedPlugin interface { + Acquire() + Release() + CompatPlugin +} + // PluginGetter is the interface implemented by Store type PluginGetter interface { Get(name, capability string, mode int) (CompatPlugin, error) diff --git a/plugin/v2/plugin.go b/plugin/v2/plugin.go index ff8f5ff2352b61886fbc5e277bf65409ebae74fe..553dfd875c16beb19c8692af6524703b0a2df9ac 100644 --- a/plugin/v2/plugin.go +++ b/plugin/v2/plugin.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/oci" + "github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/plugins" "github.com/docker/docker/pkg/system" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -294,6 +295,19 @@ func (p *Plugin) AddRefCount(count int) { p.refCount += count } +// Acquire increments the plugin's reference count +// This should be followed up by `Release()` when the plugin is no longer in use. +func (p *Plugin) Acquire() { + p.AddRefCount(plugingetter.ACQUIRE) +} + +// Release decrements the plugin's reference count +// This should only be called when the plugin is no longer in use, e.g. with +// via `Acquire()` or getter.Get("name", "type", plugingetter.ACQUIRE) +func (p *Plugin) Release() { + p.AddRefCount(plugingetter.RELEASE) +} + // InitSpec creates an OCI spec from the plugin's config. func (p *Plugin) InitSpec(s specs.Spec) (*specs.Spec, error) { s.Root = specs.Root{ diff --git a/volume/drivers/extpoint.go b/volume/drivers/extpoint.go index 78f86948f6609daea27ad99309455f30541e9783..16ac0f3d962de61becc4062753ed5a77e5065e28 100644 --- a/volume/drivers/extpoint.go +++ b/volume/drivers/extpoint.go @@ -153,7 +153,7 @@ func CreateDriver(name string) (volume.Driver, error) { if name == "" { name = volume.DefaultDriverName } - return lookup(name, getter.CREATE) + return lookup(name, getter.ACQUIRE) } // RemoveDriver returns a volume driver by its name and decrements RefCount.. @@ -162,7 +162,7 @@ func RemoveDriver(name string) (volume.Driver, error) { if name == "" { name = volume.DefaultDriverName } - return lookup(name, getter.REMOVE) + return lookup(name, getter.RELEASE) } // GetDriverList returns list of volume drivers registered. From 20dc482b1054f0c2e0a941db5321b671b83bc7fc Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 14 Dec 2016 12:00:04 +0000 Subject: [PATCH 66/68] Add display name for tags in swagger.yaml In #29071, we made the tags the correct name for generating types, at the expense of the menu in the documentation looking good. ReDoc now has support for tag display names ( https://github.com/Rebilly/ReDoc/pull/152 ), so we can assign a more human-friendly name to the menu items. Signed-off-by: Ben Firshman (cherry picked from commit 0caa6c218c89147b2d61a1bbb87c48ad16495a11) Signed-off-by: Victor Vieux --- api/swagger.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/swagger.yaml b/api/swagger.yaml index 753dcb267e33b545641c75362be35f6582a6b02e..00757b8a650a2f51d4f997fbe3407de097763056 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -97,37 +97,49 @@ info: tags: # Primary objects - name: "Container" + x-displayName: "Containers" description: | Create and manage containers. - name: "Image" + x-displayName: "Images" - name: "Network" + x-displayName: "Networks" description: | Networks are user-defined networks that containers can be attached to. See the [networking documentation](https://docs.docker.com/engine/userguide/networking/) for more information. - name: "Volume" + x-displayName: "Volumes" description: | Create and manage persistent storage that can be attached to containers. - name: "Exec" + x-displayName: "Exec" description: | Run new commands inside running containers. See the [command-line reference](https://docs.docker.com/engine/reference/commandline/exec/) for more information. To exec a command in a container, you first need to create an exec instance, then start it. These two API endpoints are wrapped up in a single command-line command, `docker exec`. - name: "Secret" + x-displayName: "Secrets" # Swarm things - name: "Swarm" + x-displayName: "Swarm" description: | Engines can be clustered together in a swarm. See [the swarm mode documentation](https://docs.docker.com/engine/swarm/) for more information. - name: "Node" + x-displayName: "Nodes" description: | Nodes are instances of the Engine participating in a swarm. Swarm mode must be enabled for these endpoints to work. - name: "Service" + x-displayName: "Services" description: | Services are the definitions of tasks to run on a swarm. Swarm mode must be enabled for these endpoints to work. - name: "Task" + x-displayName: "Tasks" description: | A task is a container running on a swarm. It is the atomic scheduling unit of swarm. Swarm mode must be enabled for these endpoints to work. # System things - name: "Plugin" + x-displayName: "Plugins" - name: "System" + x-displayName: "System" definitions: Port: From a4af72fc87aaa4c7ed24806ffc5f7d0a33ad5e11 Mon Sep 17 00:00:00 2001 From: yuexiao-wang Date: Wed, 14 Dec 2016 18:22:59 +0800 Subject: [PATCH 67/68] Update the manual for docker wait Signed-off-by: yuexiao-wang (cherry picked from commit 303ff807f2690c1f572439bb5bc53a9ba89bf48f) Signed-off-by: Victor Vieux --- man/docker-wait.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/docker-wait.1.md b/man/docker-wait.1.md index 5f07bacc0e2c0742e24c659574980a71ae1c4f0f..678800966b4250f6b583033569d5e843a2794b19 100644 --- a/man/docker-wait.1.md +++ b/man/docker-wait.1.md @@ -2,7 +2,7 @@ % Docker Community % JUNE 2014 # NAME -docker-wait - Block until a container stops, then print its exit code. +docker-wait - Block until one or more containers stop, then print their exit codes # SYNOPSIS **docker wait** @@ -11,7 +11,7 @@ CONTAINER [CONTAINER...] # DESCRIPTION -Block until a container stops, then print its exit code. +Block until one or more containers stop, then print their exit codes. # OPTIONS **--help** From efdf7ae8f3bab9620f8ebd99a978e8f494aed656 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 13 Dec 2016 17:46:01 -0800 Subject: [PATCH 68/68] skip empty networks in plugin install Signed-off-by: Victor Vieux (cherry picked from commit 04e35a01fcc7292272d688599ed23d829b85f46b) Signed-off-by: Victor Vieux --- plugin/backend_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/backend_linux.go b/plugin/backend_linux.go index 416edc34774fb7b49f3cab02bef9d92c0a7af9d6..739f7b83cd213e8c568b4402119338376341aeba 100644 --- a/plugin/backend_linux.go +++ b/plugin/backend_linux.go @@ -125,7 +125,7 @@ func computePrivileges(pd distribution.PullData) (types.PluginPrivileges, error) } var privileges types.PluginPrivileges - if c.Network.Type != "null" && c.Network.Type != "bridge" { + if c.Network.Type != "null" && c.Network.Type != "bridge" && c.Network.Type != "" { privileges = append(privileges, types.PluginPrivilege{ Name: "network", Description: "permissions to access a network",