Merge pull request #316 from thaJeztah/19.03_backport_buildkit_userns_remap_take2

[19.03 backport] builder-next: userns remap support and honor daemon's DNS config

builder/builder-next/adapters/snapshot/snapshot.go

@@ -29,6 +29,7 @@ type Opt struct {
 	GraphDriver     graphdriver.Driver
 	LayerStore      layer.Store
 	Root            string
+	IdentityMapping *idtools.IdentityMapping
 }
 
 type graphIDRegistrar interface {
@@ -79,7 +80,7 @@ func (s *snapshotter) Name() string {
 }
 
 func (s *snapshotter) IdentityMapping() *idtools.IdentityMapping {
-	return nil
+	return s.opt.IdentityMapping
 }
 
 func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
@@ -253,6 +254,7 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl
 		id := identity.NewID()
 		var rwlayer layer.RWLayer
 		return &mountable{
+			idmap: s.opt.IdentityMapping,
 			acquire: func() ([]mount.Mount, error) {
 				rwlayer, err = s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil)
 				if err != nil {
@@ -278,6 +280,7 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl
 	id, _ := s.getGraphDriverID(key)
 
 	return &mountable{
+		idmap: s.opt.IdentityMapping,
 		acquire: func() ([]mount.Mount, error) {
 			rootfs, err := s.opt.GraphDriver.Get(id, "")
 			if err != nil {
@@ -440,6 +443,7 @@ type mountable struct {
 	acquire  func() ([]mount.Mount, error)
 	release  func() error
 	refCount int
+	idmap    *idtools.IdentityMapping
 }
 
 func (m *mountable) Mount() ([]mount.Mount, error) {
@@ -480,5 +484,5 @@ func (m *mountable) Release() error {
 }
 
 func (m *mountable) IdentityMapping() *idtools.IdentityMapping {
-	return nil
+	return m.idmap
 }

builder/builder-next/builder.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/images"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork"
@@ -73,6 +74,8 @@ type Opt struct {
 	ResolverOpt         resolver.ResolveOptionsFunc
 	BuilderConfig       config.BuilderConfig
 	Rootless            bool
+	IdentityMapping     *idtools.IdentityMapping
+	DNSConfig           config.DNSConfig
 }
 
 // Builder can build using BuildKit backend
@@ -88,6 +91,10 @@ type Builder struct {
 func New(opt Opt) (*Builder, error) {
 	reqHandler := newReqBodyHandler(tracing.DefaultTransport)
 
+	if opt.IdentityMapping != nil && opt.IdentityMapping.Empty() {
+		opt.IdentityMapping = nil
+	}
+
 	c, err := newController(reqHandler, opt)
 	if err != nil {
 		return nil, err

builder/builder-next/controller.go

@@ -38,7 +38,7 @@ import (
 )
 
 func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
-	if err := os.MkdirAll(opt.Root, 0700); err != nil {
+	if err := os.MkdirAll(opt.Root, 0711); err != nil {
 		return nil, err
 	}
 
@@ -58,6 +58,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		GraphDriver:     driver,
 		LayerStore:      dist.LayerStore,
 		Root:            root,
+		IdentityMapping: opt.IdentityMapping,
 	})
 	if err != nil {
 		return nil, err
@@ -112,7 +113,9 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		return nil, err
 	}
 
-	exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, opt.Rootless)
+	dns := getDNSConfig(opt.DNSConfig)
+
+	exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping)
 	if err != nil {
 		return nil, err
 	}

builder/builder-next/executor_unix.go

@@ -8,8 +8,11 @@ import (
 	"strconv"
 	"sync"
 
+	"github.com/docker/docker/daemon/config"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/libnetwork"
 	"github.com/moby/buildkit/executor"
+	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/executor/runcexecutor"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver/pb"
@@ -20,7 +23,7 @@ import (
 
 const networkName = "bridge"
 
-func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, rootless bool) (executor.Executor, error) {
+func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, dnsConfig *oci.DNSConfig, rootless bool, idmap *idtools.IdentityMapping) (executor.Executor, error) {
 	networkProviders := map[pb.NetMode]network.Provider{
 		pb.NetMode_UNSET: &bridgeProvider{NetworkController: net, Root: filepath.Join(root, "net")},
 		pb.NetMode_HOST:  network.NewHostProvider(),
@@ -32,6 +35,8 @@ func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, ro
 		DefaultCgroupParent: cgroupParent,
 		Rootless:            rootless,
 		NoPivot:             os.Getenv("DOCKER_RAMDISK") != "",
+		IdentityMapping:     idmap,
+		DNS:                 dnsConfig,
 	}, networkProviders)
 }
 
@@ -115,3 +120,14 @@ func (iface *lnInterface) Close() error {
 	}
 	return iface.err
 }
+
+func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
+	if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
+		return &oci.DNSConfig{
+			Nameservers:   cfg.DNS,
+			SearchDomains: cfg.DNSSearch,
+			Options:       cfg.DNSOptions,
+		}
+	}
+	return nil
+}

builder/builder-next/executor_windows.go

@@ -5,12 +5,15 @@ import (
 	"errors"
 	"io"
 
+	"github.com/docker/docker/daemon/config"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/libnetwork"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/executor"
+	"github.com/moby/buildkit/executor/oci"
 )
 
-func newExecutor(_, _ string, _ libnetwork.NetworkController, _ bool) (executor.Executor, error) {
+func newExecutor(_, _ string, _ libnetwork.NetworkController, _ *oci.DNSConfig, _ bool, _ *idtools.IdentityMapping) (executor.Executor, error) {
 	return &winExecutor{}, nil
 }
 
@@ -20,3 +23,7 @@ type winExecutor struct {
 func (e *winExecutor) Exec(ctx context.Context, meta executor.Meta, rootfs cache.Mountable, mounts []executor.Mount, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
 	return errors.New("buildkit executor not implemented for windows")
 }
+
+func getDNSConfig(config.DNSConfig) *oci.DNSConfig {
+	return nil
+}

cmd/dockerd/daemon.go

@@ -318,6 +318,8 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
 		ResolverOpt:         d.NewResolveOptionsFunc(),
 		BuilderConfig:       config.Builder,
 		Rootless:            d.Rootless(),
+		IdentityMapping:     d.IdentityMapping(),
+		DNSConfig:           config.DNSConfig,
 	})
 	if err != nil {
 		return opts, err

daemon/config/config.go

@@ -109,6 +109,13 @@ type CommonTLSOptions struct {
 	KeyFile  string `json:"tlskey,omitempty"`
 }
 
+// DNSConfig defines the DNS configurations.
+type DNSConfig struct {
+	DNS        []string `json:"dns,omitempty"`
+	DNSOptions []string `json:"dns-opts,omitempty"`
+	DNSSearch  []string `json:"dns-search,omitempty"`
+}
+
 // CommonConfig defines the configuration of a docker daemon which is
 // common across platforms.
 // It includes json tags to deserialize configuration from a file
@@ -119,9 +126,6 @@ type CommonConfig struct {
 	AutoRestart           bool                      `json:"-"`
 	Context               map[string][]string       `json:"-"`
 	DisableBridge         bool                      `json:"-"`
-	DNS                   []string                  `json:"dns,omitempty"`
-	DNSOptions            []string                  `json:"dns-opts,omitempty"`
-	DNSSearch             []string                  `json:"dns-search,omitempty"`
 	ExecOptions           []string                  `json:"exec-opts,omitempty"`
 	GraphDriver           string                    `json:"storage-driver,omitempty"`
 	GraphOptions          []string                  `json:"storage-opts,omitempty"`
@@ -200,6 +204,7 @@ type CommonConfig struct {
 
 	MetricsAddress string `json:"metrics-addr"`
 
+	DNSConfig
 	LogConfig
 	BridgeConfig // bridgeConfig holds bridge network specific configuration.
 	NetworkConfig

daemon/config/config_test.go

@@ -244,31 +244,39 @@ func TestValidateConfigurationErrors(t *testing.T) {
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNS: []string{"1.1.1.1o"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNS: []string{"2.2.2.2", "1.1.1.1o"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNSSearch: []string{"123456"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNSSearch: []string{"a.b.c", "123456"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
@@ -329,17 +337,21 @@ func TestValidateConfiguration(t *testing.T) {
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNS: []string{"1.1.1.1"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{
+					DNSConfig: DNSConfig{
 						DNSSearch: []string{"a.b.c"},
 					},
 				},
 			},
+		},
 		{
 			config: &Config{
 				CommonConfig: CommonConfig{

hack/dockerfile/install/vndr.installer

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-VNDR_COMMIT=81cb8916aad3c8d06193f008dba3e16f82851f52
+VNDR_COMMIT=f5ab8fc5fb64d66b5c6e55a0bcb58b2e92362fa0
 
 install_vndr() {
 	echo "Install vndr version $VNDR_COMMIT"

vendor/github.com/gogo/googleapis/go.mod

@@ -0,0 +1,5 @@
+module github.com/gogo/googleapis
+
+go 1.12
+
+require github.com/gogo/protobuf v1.2.1

vendor/github.com/gogo/protobuf/go.mod

@@ -0,0 +1,3 @@
+module github.com/gogo/protobuf
+
+require github.com/kisielk/errcheck v1.1.0 // indirect

vendor/github.com/google/uuid/go.mod

@@ -0,0 +1 @@
+module github.com/google/uuid

vendor/github.com/gorilla/mux/go.mod

@@ -0,0 +1 @@
+module github.com/gorilla/mux

vendor/github.com/hashicorp/golang-lru/go.mod

@@ -0,0 +1 @@
+module github.com/hashicorp/golang-lru

vendor/github.com/konsorten/go-windows-terminal-sequences/go.mod

@@ -0,0 +1 @@
+module github.com/konsorten/go-windows-terminal-sequences

vendor/github.com/kr/pty/go.mod

@@ -0,0 +1 @@
+module github.com/kr/pty

vendor/github.com/mattn/go-shellwords/go.mod

@@ -0,0 +1 @@
+module github.com/mattn/go-shellwords

vendor/github.com/moby/buildkit/go.mod

@@ -0,0 +1,77 @@
+module github.com/moby/buildkit
+
+go 1.11
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/Microsoft/go-winio v0.4.13-0.20190408173621-84b4ab48a507
+	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 // indirect
+	github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect
+	github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
+	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
+	github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0
+	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
+	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect
+	github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645
+	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
+	github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 // indirect
+	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd // indirect
+	github.com/containernetworking/cni v0.6.1-0.20180218032124-142cde0c766c // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
+	github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0
+	github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
+	github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c
+	github.com/docker/docker-credential-helpers v0.6.0 // indirect
+	github.com/docker/go-connections v0.3.0
+	github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect
+	github.com/docker/libnetwork v0.8.0-dev.2.0.20190604151032-3c26b4e7495e
+	github.com/godbus/dbus v4.1.0+incompatible // indirect
+	github.com/gofrs/flock v0.7.0
+	github.com/gogo/googleapis v1.1.0
+	github.com/gogo/protobuf v1.2.0
+	github.com/golang/protobuf v1.2.0
+	github.com/google/go-cmp v0.2.0
+	github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
+	github.com/hashicorp/go-immutable-radix v1.0.0
+	github.com/hashicorp/golang-lru v0.0.0-20160207214719-a0d98a5f2880
+	github.com/hashicorp/uuid v0.0.0-20160311170451-ebb0a03e909c // indirect
+	github.com/ishidawataru/sctp v0.0.0-20180213033435-07191f837fed // indirect
+	github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452
+	github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c
+	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/runc v1.0.0-rc8
+	github.com/opencontainers/runtime-spec v0.0.0-20180909173843-eba862dc2470
+	github.com/opentracing-contrib/go-stdlib v0.0.0-20171029140428-b1a47cfbdd75
+	github.com/opentracing/opentracing-go v0.0.0-20171003133519-1361b9cd60be
+	github.com/pkg/errors v0.8.1
+	github.com/pkg/profile v1.2.1
+	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
+	github.com/sirupsen/logrus v1.3.0
+	github.com/stretchr/testify v1.3.0
+	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
+	github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
+	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
+	github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
+	github.com/uber/jaeger-lib v1.2.1 // indirect
+	github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5
+	github.com/vishvananda/netlink v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
+	go.etcd.io/bbolt v1.3.2
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
+	golang.org/x/sys v0.0.0-20190303122642-d455e41777fc
+	golang.org/x/time v0.0.0-20161028155119-f51c12702a4d
+	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
+	google.golang.org/grpc v1.20.1
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gotest.tools v2.2.0+incompatible
+)
+
+replace github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
+
+replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305

vendor/github.com/sirupsen/logrus/go.mod

@@ -0,0 +1,10 @@
+module github.com/sirupsen/logrus
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.1
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	github.com/stretchr/testify v1.2.2
+	golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33
+)

vendor/github.com/tonistiigi/fsutil/go.mod

@@ -0,0 +1,28 @@
+module github.com/tonistiigi/fsutil
+
+require (
+	github.com/Microsoft/go-winio v0.4.11 // indirect
+	github.com/Microsoft/hcsshim v0.8.5 // indirect
+	github.com/containerd/containerd v1.2.4
+	github.com/containerd/continuity v0.0.0-20181001140422-bd77b46c8352
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/docker v0.0.0-20180531152204-71cd53e4a197
+	github.com/docker/go-units v0.3.1 // indirect
+	github.com/gogo/protobuf v1.0.0
+	github.com/google/go-cmp v0.2.0 // indirect
+	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
+	github.com/onsi/ginkgo v1.7.0 // indirect
+	github.com/onsi/gomega v1.4.3 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/runc v1.0.0-rc6 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/sirupsen/logrus v1.0.3 // indirect
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20190129210102-0709b304e793 // indirect
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
+	golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e
+	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
+	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+	gotest.tools v2.1.0+incompatible // indirect
+)

vendor/golang.org/x/crypto/go.mod

@@ -0,0 +1,3 @@
+module golang.org/x/crypto
+
+require golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e

vendor/golang.org/x/net/go.mod

@@ -0,0 +1,6 @@
+module golang.org/x/net
+
+require (
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/text v0.3.0
+)

vendor/golang.org/x/sync/go.mod

@@ -0,0 +1 @@
+module golang.org/x/sync

vendor/golang.org/x/sys/go.mod

@@ -0,0 +1,3 @@
+module golang.org/x/sys
+
+go 1.12

vendor/google.golang.org/grpc/go.mod

@@ -0,0 +1,19 @@
+module google.golang.org/grpc
+
+require (
+	cloud.google.com/go v0.26.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/client9/misspell v0.3.4
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/golang/mock v1.1.1
+	github.com/golang/protobuf v1.2.0
+	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
+	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
+	golang.org/x/tools v0.0.0-20190311212946-11955173bddd
+	google.golang.org/appengine v1.1.0 // indirect
+	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
+	honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099
+)

vendor/gotest.tools/go.mod

@@ -0,0 +1,8 @@
+module gotest.tools
+
+require (
+	github.com/google/go-cmp v0.2.0
+	github.com/pkg/errors v0.8.0
+	github.com/spf13/pflag v1.0.3
+	golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d
+)