Merge pull request #31055 from Microsoft/jjh/fix30278
Windows: Retry workaround for RS1/RS2 compute system enumeration
This commit is contained in:
Victor Vieux
GitHub
commit
7761c69e23
4 changed files with 55 additions and 33 deletions
|
|
@ -31,6 +31,7 @@ import (
|
|||
"github.com/docker/docker/pkg/ioutils"
|
||||
"github.com/docker/docker/pkg/longpath"
|
||||
"github.com/docker/docker/pkg/reexec"
|
||||
"github.com/docker/docker/pkg/system"
|
||||
units "github.com/docker/go-units"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
|
@ -265,19 +266,35 @@ func (d *Driver) Remove(id string) error {
|
|||
// it is a transient error. Retry until it succeeds.
|
||||
var computeSystems []hcsshim.ContainerProperties
|
||||
retryCount := 0
|
||||
osv := system.GetOSVersion()
|
||||
for {
|
||||
// Get and terminate any template VMs that are currently using the layer
|
||||
// Get and terminate any template VMs that are currently using the layer.
|
||||
// Note: It is unfortunate that we end up in the graphdrivers Remove() call
|
||||
// for both containers and images, but the logic for template VMs is only
|
||||
// needed for images - specifically we are looking to see if a base layer
|
||||
// is in use by a template VM as a result of having started a Hyper-V
|
||||
// container at some point.
|
||||
//
|
||||
// We have a retry loop for ErrVmcomputeOperationInvalidState and
|
||||
// ErrVmcomputeOperationAccessIsDenied as there is a race condition
|
||||
// in RS1 and RS2 building during enumeration when a silo is going away
|
||||
// for example under it, in HCS. AccessIsDenied added to fix 30278.
|
||||
//
|
||||
// TODO @jhowardmsft - For RS3, we can remove the retries. Also consider
|
||||
// using platform APIs (if available) to get this more succinctly. Also
|
||||
// consider enlighting the Remove() interface to have context of why
|
||||
// the remove is being called - that could improve efficiency by not
|
||||
// enumerating compute systems during a remove of a container as it's
|
||||
// not required.
|
||||
computeSystems, err = hcsshim.GetContainers(hcsshim.ComputeSystemQuery{})
|
||||
if err != nil {
|
||||
if err == hcsshim.ErrVmcomputeOperationInvalidState {
|
||||
if retryCount >= 5 {
|
||||
// If we are unable to get the list of containers
|
||||
// go ahead and attempt to delete the layer anyway
|
||||
// as it will most likely work.
|
||||
if (osv.Build < 15139) &&
|
||||
((err == hcsshim.ErrVmcomputeOperationInvalidState) || (err == hcsshim.ErrVmcomputeOperationAccessIsDenied)) {
|
||||
if retryCount >= 500 {
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
time.Sleep(2 * time.Second)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# the following lines are in sorted order, FYI
|
||||
github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
|
||||
github.com/Microsoft/hcsshim v0.5.10
|
||||
github.com/Microsoft/hcsshim v0.5.12
|
||||
github.com/Microsoft/go-winio v0.3.8
|
||||
github.com/Sirupsen/logrus v0.11.0
|
||||
github.com/davecgh/go-spew 6d212800a42e8ab5c146b8ace3490ee17e5225f9
|
||||
|
|
|
|||
4
vendor/github.com/Microsoft/hcsshim/errors.go
generated
vendored
4
vendor/github.com/Microsoft/hcsshim/errors.go
generated
vendored
|
|
@ -50,6 +50,10 @@ var (
|
|||
|
||||
// ErrProcNotFound is an error encountered when the the process cannot be found
|
||||
ErrProcNotFound = syscall.Errno(0x7f)
|
||||
|
||||
// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
|
||||
// builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3.
|
||||
ErrVmcomputeOperationAccessIsDenied = syscall.Errno(0x5)
|
||||
)
|
||||
|
||||
// ProcessError is an error encountered in HCS during an operation on a Process object
|
||||
|
|
|
|||
51
vendor/github.com/Microsoft/hcsshim/interface.go
generated
vendored
51
vendor/github.com/Microsoft/hcsshim/interface.go
generated
vendored
|
|
@ -41,31 +41,32 @@ type HvRuntime struct {
|
|||
// ContainerConfig is used as both the input of CreateContainer
|
||||
// and to convert the parameters to JSON for passing onto the HCS
|
||||
type ContainerConfig struct {
|
||||
SystemType string // HCS requires this to be hard-coded to "Container"
|
||||
Name string // Name of the container. We use the docker ID.
|
||||
Owner string // The management platform that created this container
|
||||
IsDummy bool // Used for development purposes.
|
||||
VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
|
||||
IgnoreFlushesDuringBoot bool // Optimization hint for container startup in Windows
|
||||
LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID
|
||||
Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
|
||||
Credentials string `json:",omitempty"` // Credentials information
|
||||
ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container.
|
||||
ProcessorWeight uint64 `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default.
|
||||
ProcessorMaximum int64 `json:",omitempty"` // CPU maximum usage percent 1..100
|
||||
StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS
|
||||
StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
|
||||
StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
|
||||
MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes
|
||||
HostName string // Hostname
|
||||
MappedDirectories []MappedDir // List of mapped directories (volumes/mounts)
|
||||
SandboxPath string `json:",omitempty"` // Location of unmounted sandbox. Used by Hyper-V containers only. Format %root%\windowsfilter
|
||||
HvPartition bool // True if it a Hyper-V Container
|
||||
EndpointList []string // List of networking endpoints to be attached to container
|
||||
HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
|
||||
Servicing bool // True if this container is for servicing
|
||||
AllowUnqualifiedDNSQuery bool // True to allow unqualified DNS name resolution
|
||||
DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
|
||||
SystemType string // HCS requires this to be hard-coded to "Container"
|
||||
Name string // Name of the container. We use the docker ID.
|
||||
Owner string // The management platform that created this container
|
||||
IsDummy bool // Used for development purposes.
|
||||
VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
|
||||
IgnoreFlushesDuringBoot bool // Optimization hint for container startup in Windows
|
||||
LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID
|
||||
Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
|
||||
Credentials string `json:",omitempty"` // Credentials information
|
||||
ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container.
|
||||
ProcessorWeight uint64 `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default.
|
||||
ProcessorMaximum int64 `json:",omitempty"` // CPU maximum usage percent 1..100
|
||||
StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS
|
||||
StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
|
||||
StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
|
||||
MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes
|
||||
HostName string // Hostname
|
||||
MappedDirectories []MappedDir // List of mapped directories (volumes/mounts)
|
||||
SandboxPath string `json:",omitempty"` // Location of unmounted sandbox. Used by Hyper-V containers only. Format %root%\windowsfilter
|
||||
HvPartition bool // True if it a Hyper-V Container
|
||||
EndpointList []string // List of networking endpoints to be attached to container
|
||||
NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with.
|
||||
HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
|
||||
Servicing bool // True if this container is for servicing
|
||||
AllowUnqualifiedDNSQuery bool // True to allow unqualified DNS name resolution
|
||||
DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
|
||||
}
|
||||
|
||||
type ComputeSystemQuery struct {
|
||||
|
|
|
|||
Loading…
Reference in a new issue