Merge pull request #37674 from Microsoft/jjh/revendor82018

Revendor Microsoft/hcsshim and go-winio, plus container/containerd

daemon/daemon.go

@@ -741,7 +741,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
 	}
 	if config.ContainerdAddr != "" {
-		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts))
+		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 		}
@@ -753,7 +753,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		// Windows is not currently using containerd, keep the
 		// client as nil
 		if config.ContainerdAddr != "" {
-			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts))
+			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 			}

libcontainerd/supervisor/remote_daemon.go

@@ -268,7 +268,7 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 				continue
 			}
 
-			client, err = containerd.New(r.GRPC.Address)
+			client, err = containerd.New(r.GRPC.Address, containerd.WithTimeout(60*time.Second))
 			if err != nil {
 				r.logger.WithError(err).Error("failed connecting to containerd")
 				delay = time.After(100 * time.Millisecond)

vendor.conf

@@ -1,7 +1,7 @@
 # the following lines are in sorted order, FYI
 github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
-github.com/Microsoft/hcsshim v0.6.12
+github.com/Microsoft/hcsshim v0.6.14
-github.com/Microsoft/go-winio v0.4.9
+github.com/Microsoft/go-winio v0.4.10
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git
 github.com/golang/gddo 9b12a26f3fbd7397dee4e20939ddca719d840d2a
@@ -114,7 +114,7 @@ github.com/googleapis/gax-go v2.0.0
 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9
 
 # containerd
-github.com/containerd/containerd v1.2.0-beta.0
+github.com/containerd/containerd 3f42445e38d1081f4b8c3b8d7d1ed1860198ed7a
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/containerd/continuity d3c23511c1bf5851696cba83143d9cbcd666869b
 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2

vendor/github.com/Microsoft/go-winio/backuptar/tar.go

@@ -303,7 +303,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		if err != nil {
 			return "", 0, nil, err
 		}
-		fileInfo.FileAttributes = uintptr(attr)
+		fileInfo.FileAttributes = uint32(attr)
 	} else {
 		if hdr.Typeflag == tar.TypeDir {
 			fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY

vendor/github.com/Microsoft/go-winio/fileinfo.go

@@ -20,7 +20,8 @@ const (
 // FileBasicInfo contains file access time and file attributes information.
 type FileBasicInfo struct {
 	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
-	FileAttributes                                          uintptr // includes padding
+	FileAttributes                                          uint32
+	pad                                                     uint32 // padding
 }
 
 // GetFileBasicInfo retrieves times and attributes for a file.

vendor/github.com/Microsoft/hcsshim/container.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strconv"
 	"sync"
 	"syscall"
 	"time"
@@ -136,8 +137,26 @@ type ResourceModificationRequestResponse struct {
 // is merged in the CreateContainer call to HCS.
 var createContainerAdditionalJSON string
 
+// currentContainerStarts is used to limit the number of concurrent container
+// starts.
+var currentContainerStarts containerStarts
+
+type containerStarts struct {
+	maxParallel int
+	inProgress  int
+	sync.Mutex
+}
+
 func init() {
 	createContainerAdditionalJSON = os.Getenv("HCSSHIM_CREATECONTAINER_ADDITIONALJSON")
+	mpsS := os.Getenv("HCSSHIM_MAX_PARALLEL_START")
+	if len(mpsS) > 0 {
+		mpsI, err := strconv.Atoi(mpsS)
+		if err != nil || mpsI < 0 {
+			return
+		}
+		currentContainerStarts.maxParallel = mpsI
+	}
 }
 
 // CreateContainer creates a new container with the given configuration but does not start it.
@@ -325,6 +344,32 @@ func (container *container) Start() error {
 		return makeContainerError(container, operation, "", ErrAlreadyClosed)
 	}
 
+	// This is a very simple backoff-retry loop to limit the number
+	// of parallel container starts if environment variable
+	// HCSSHIM_MAX_PARALLEL_START is set to a positive integer.
+	// It should generally only be used as a workaround to various
+	// platform issues that exist between RS1 and RS4 as of Aug 2018.
+	if currentContainerStarts.maxParallel > 0 {
+		for {
+			currentContainerStarts.Lock()
+			if currentContainerStarts.inProgress < currentContainerStarts.maxParallel {
+				currentContainerStarts.inProgress++
+				currentContainerStarts.Unlock()
+				break
+			}
+			if currentContainerStarts.inProgress == currentContainerStarts.maxParallel {
+				currentContainerStarts.Unlock()
+				time.Sleep(100 * time.Millisecond)
+			}
+		}
+		// Make sure we decrement the count when we are done.
+		defer func() {
+			currentContainerStarts.Lock()
+			currentContainerStarts.inProgress--
+			currentContainerStarts.Unlock()
+		}()
+	}
+
 	var resultp *uint16
 	err := hcsStartComputeSystem(container.handle, "", &resultp)
 	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemStartCompleted, &defaultTimeout)

vendor/github.com/Microsoft/hcsshim/interface.go

@@ -6,6 +6,30 @@ import (
 	"time"
 )
 
+// RegistryKey is used to specify registry key name
+type RegistryKey struct {
+	Hive     string
+	Name     string
+	Volatile bool `json:",omitempty"`
+}
+
+// RegistryKey is used to specify registry key name
+type RegistryValue struct {
+	Key         RegistryKey
+	Name        string
+	Type        string
+	StringValue string  `json:",omitempty"`
+	BinaryValue []byte  `json:",omitempty"`
+	DWordValue  *uint32 `json:",omitempty"`
+	QWordValue  *uint64 `json:",omitempty"`
+	CustomType  *uint32 `json:",omitempty"`
+}
+
+type RegistryChanges struct {
+	AddValues  []RegistryValue `json:",omitempty"`
+	DeleteKeys []RegistryValue `json:",omitempty"`
+}
+
 // ProcessConfig is used as both the input of Container.CreateProcess
 // and to convert the parameters to JSON for passing onto the HCS
 type ProcessConfig struct {
@@ -104,6 +128,7 @@ type ContainerConfig struct {
 	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
 	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start
 	AssignedDevices             []AssignedDevice    `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5
+	RegistryChanges             *RegistryChanges    `json:",omitempty"` // Registry changes to be applied to the container
 }
 
 type ComputeSystemQuery struct {

vendor/github.com/Microsoft/hcsshim/legacy.go

@@ -283,7 +283,7 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 		if err != nil {
 			return
 		}
-		fileInfo.FileAttributes = uintptr(attr)
+		fileInfo.FileAttributes = attr
 		beginning := int64(4)
 
 		// Find the accurate file size.

vendor/github.com/containerd/containerd/archive/tar_windows.go

@@ -259,7 +259,7 @@ func fileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		if err != nil {
 			return "", 0, nil, err
 		}
-		fileInfo.FileAttributes = uintptr(attr)
+		fileInfo.FileAttributes = uint32(attr)
 	} else {
 		if hdr.Typeflag == tar.TypeDir {
 			fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY

vendor/github.com/containerd/containerd/client.go

@@ -82,6 +82,9 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 			return nil, err
 		}
 	}
+	if copts.timeout == 0 {
+		copts.timeout = 10 * time.Second
+	}
 	rt := fmt.Sprintf("%s.%s", plugin.RuntimePlugin, runtime.GOOS)
 	if copts.defaultRuntime != "" {
 		rt = copts.defaultRuntime
@@ -115,7 +118,7 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 			)
 		}
 		connector := func() (*grpc.ClientConn, error) {
-			ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+			ctx, cancel := context.WithTimeout(context.Background(), copts.timeout)
 			defer cancel()
 			conn, err := grpc.DialContext(ctx, dialer.DialAddress(address), gopts...)
 			if err != nil {

vendor/github.com/containerd/containerd/client_opts.go

@@ -17,6 +17,8 @@
 package containerd
 
 import (
+	"time"
+
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
@@ -28,6 +30,7 @@ type clientOpts struct {
 	defaultRuntime string
 	services       *services
 	dialOptions    []grpc.DialOption
+	timeout        time.Duration
 }
 
 // ClientOpt allows callers to set options on the containerd client
@@ -71,6 +74,14 @@ func WithServices(opts ...ServicesOpt) ClientOpt {
 	}
 }
 
+// WithTimeout sets the connection timeout for the client
+func WithTimeout(d time.Duration) ClientOpt {
+	return func(c *clientOpts) error {
+		c.timeout = d
+		return nil
+	}
+}
+
 // RemoteOpt allows the caller to set distribution options for a remote
 type RemoteOpt func(*Client, *RemoteContext) error
 

vendor/github.com/containerd/containerd/vendor.conf

@@ -20,7 +20,7 @@ github.com/gogo/protobuf v1.0.0
 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef
 github.com/golang/protobuf v1.1.0
 github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce # v1.0.1-43-gd810dbc
-github.com/opencontainers/runc 69663f0bd4b60df09991c08812a60108003fa340
+github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd
 github.com/sirupsen/logrus v1.0.0
 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c
 golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac
@@ -32,8 +32,8 @@ github.com/opencontainers/image-spec v1.0.1
 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
-github.com/Microsoft/go-winio v0.4.7
+github.com/Microsoft/go-winio v0.4.10
-github.com/Microsoft/hcsshim v0.6.11
+github.com/Microsoft/hcsshim v0.6.14
 github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4