Merge pull request #33089 from Microsoft/jjh/hccsshimv0.5.16

Revendor Microsoft\hcsshim @ v0.5.17

vendor.conf

@@ -1,6 +1,6 @@
 # the following lines are in sorted order, FYI
 github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
-github.com/Microsoft/hcsshim v0.5.13
+github.com/Microsoft/hcsshim v0.5.17
 github.com/Microsoft/go-winio v0.3.9
 github.com/Sirupsen/logrus v0.11.0
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76

vendor/github.com/Microsoft/hcsshim/LICENSE

@@ -18,5 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
+SOFTWARE.

vendor/github.com/Microsoft/hcsshim/container.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -105,6 +104,27 @@ type ProcessListItem struct {
 	UserTime100ns                uint64    `json:",omitempty"`
 }
 
+// Type of Request Support in ModifySystem
+type RequestType string
+
+// Type of Resource Support in ModifySystem
+type ResourceType string
+
+// RequestType const
+const (
+	Add     RequestType  = "Add"
+	Remove  RequestType  = "Remove"
+	Network ResourceType = "Network"
+)
+
+// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system
+// Supported resource types are Network and Request Types are Add/Remove
+type ResourceModificationRequestResponse struct {
+	Resource ResourceType `json:"ResourceType"`
+	Data     string       `json:"Settings"`
+	Request  RequestType  `json:"RequestType,omitempty"`
+}
+
 // createContainerAdditionalJSON is read from the environment at initialisation
 // time. It allows an environment variable to define additional JSON which
 // is merged in the CreateContainer call to HCS.
@@ -185,7 +205,6 @@ func createContainerWithJSON(id string, c *ContainerConfig, additionalJSON strin
 	}
 
 	logrus.Debugf(title+" succeeded id=%s handle=%d", id, container.handle)
-	runtime.SetFinalizer(container, closeContainer)
 	return container, nil
 }
 
@@ -243,7 +262,6 @@ func OpenContainer(id string) (Container, error) {
 	}
 
 	logrus.Debugf(title+" succeeded id=%s handle=%d", id, handle)
-	runtime.SetFinalizer(container, closeContainer)
 	return container, nil
 }
 
@@ -568,7 +586,6 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {
 	}
 
 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)
-	runtime.SetFinalizer(process, closeProcess)
 	return process, nil
 }
 
@@ -605,7 +622,6 @@ func (container *container) OpenProcess(pid int) (Process, error) {
 	}
 
 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)
-	runtime.SetFinalizer(process, closeProcess)
 	return process, nil
 }
 
@@ -631,17 +647,11 @@ func (container *container) Close() error {
 	}
 
 	container.handle = 0
-	runtime.SetFinalizer(container, nil)
 
 	logrus.Debugf(title+" succeeded id=%s", container.id)
 	return nil
 }
 
-// closeContainer wraps container.Close for use by a finalizer
-func closeContainer(container *container) {
-	container.Close()
-}
-
 func (container *container) registerCallback() error {
 	context := &notifcationWatcherContext{
 		channels: newChannels(),
@@ -698,3 +708,32 @@ func (container *container) unregisterCallback() error {
 
 	return nil
 }
+
+// Modifies the System by sending a request to HCS
+func (container *container) Modify(config *ResourceModificationRequestResponse) error {
+	container.handleLock.RLock()
+	defer container.handleLock.RUnlock()
+	operation := "Modify"
+	title := "HCSShim::Container::" + operation
+
+	if container.handle == 0 {
+		return makeContainerError(container, operation, "", ErrAlreadyClosed)
+	}
+
+	requestJSON, err := json.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	requestString := string(requestJSON)
+	logrus.Debugf(title+" id=%s request=%s", container.id, requestString)
+
+	var resultp *uint16
+	err = hcsModifyComputeSystem(container.handle, requestString, &resultp)
+	err = processHcsResult(err, resultp)
+	if err != nil {
+		return makeContainerError(container, operation, "", err)
+	}
+	logrus.Debugf(title+" succeeded id=%s", container.id)
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/errors.go

@@ -13,6 +13,13 @@ var (
 	// ErrElementNotFound is an error encountered when the object being referenced does not exist
 	ErrElementNotFound = syscall.Errno(0x490)
 
+	// ErrElementNotFound is an error encountered when the object being referenced does not exist
+	ErrNotSupported = syscall.Errno(0x32)
+
+	// ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported
+	// decimal -2147024883 / hex 0x8007000d
+	ErrInvalidData = syscall.Errno(0xd)
+
 	// ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed
 	ErrHandleClose = errors.New("hcsshim: the handle generating this notification has been closed")
 
@@ -54,6 +61,15 @@ var (
 	// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
 	// builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3.
 	ErrVmcomputeOperationAccessIsDenied = syscall.Errno(0x5)
+
+	// ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management
+	ErrVmcomputeInvalidJSON = syscall.Errno(0xc037010d)
+
+	// ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message
+	ErrVmcomputeUnknownMessage = syscall.Errno(0xc037010b)
+
+	// ErrNotSupported is an error encountered when hcs doesn't support the request
+	ErrPlatformNotSupported = errors.New("unsupported platform request")
 )
 
 // ProcessError is an error encountered in HCS during an operation on a Process object
@@ -196,6 +212,20 @@ func IsAlreadyStopped(err error) bool {
 		err == ErrProcNotFound
 }
 
+// IsNotSupported returns a boolean indicating whether the error is caused by
+// unsupported platform requests
+// Note: Currently Unsupported platform requests can be mean either
+// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage
+// is thrown from the Platform
+func IsNotSupported(err error) bool {
+	err = getInnerError(err)
+	// If Platform doesn't recognize or support the request sent, below errors are seen
+	return err == ErrVmcomputeInvalidJSON ||
+		err == ErrInvalidData ||
+		err == ErrNotSupported ||
+		err == ErrVmcomputeUnknownMessage
+}
+
 func getInnerError(err error) error {
 	switch pe := err.(type) {
 	case nil:

vendor/github.com/Microsoft/hcsshim/exportlayer.go

@@ -4,7 +4,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"runtime"
 	"syscall"
 
 	"github.com/Microsoft/go-winio"
@@ -143,7 +142,6 @@ func NewLayerReader(info DriverInfo, layerID string, parentLayerPaths []string)
 	if err != nil {
 		return nil, makeError(err, "ExportLayerBegin", "")
 	}
-	runtime.SetFinalizer(r, func(r *FilterLayerReader) { r.Close() })
 	return r, err
 }
 

vendor/github.com/Microsoft/hcsshim/hnsfuncs.go

@@ -98,6 +98,8 @@ type hnsResponse struct {
 
 func hnsCall(method, path, request string, returnResponse interface{}) error {
 	var responseBuffer *uint16
+	logrus.Debugf("[%s]=>[%s] Request : %s", method, path, request)
+
 	err := _hnsCall(method, path, request, &responseBuffer)
 	if err != nil {
 		return makeError(err, "hnsCall ", "")
@@ -158,3 +160,112 @@ func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) {
 
 	return endpoint, nil
 }
+
+// HNSListEndpointRequest makes a HNS call to query the list of available endpoints
+func HNSListEndpointRequest() ([]HNSEndpoint, error) {
+	var endpoint []HNSEndpoint
+	err := hnsCall("GET", "/endpoints/", "", &endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return endpoint, nil
+}
+
+// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container
+func HotAttachEndpoint(containerID string, endpointID string) error {
+	return modifyNetworkEndpoint(containerID, endpointID, Add)
+}
+
+// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container
+func HotDetachEndpoint(containerID string, endpointID string) error {
+	return modifyNetworkEndpoint(containerID, endpointID, Remove)
+}
+
+// ModifyContainer corresponding to the container id, by sending a request
+func modifyContainer(id string, request *ResourceModificationRequestResponse) error {
+	container, err := OpenContainer(id)
+	if err != nil {
+		if IsNotExist(err) {
+			return ErrComputeSystemDoesNotExist
+		}
+		return getInnerError(err)
+	}
+	defer container.Close()
+	err = container.Modify(request)
+	if err != nil {
+		if IsNotSupported(err) {
+			return ErrPlatformNotSupported
+		}
+		return getInnerError(err)
+	}
+
+	return nil
+}
+
+func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error {
+	requestMessage := &ResourceModificationRequestResponse{
+		Resource: Network,
+		Request:  request,
+		Data:     endpointID,
+	}
+	err := modifyContainer(containerID, requestMessage)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// GetHNSNetworkByID
+func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) {
+	return HNSNetworkRequest("GET", networkID, "")
+}
+
+// GetHNSNetworkName filtered by Name
+func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) {
+	hsnnetworks, err := HNSListNetworkRequest("GET", "", "")
+	if err != nil {
+		return nil, err
+	}
+	for _, hnsnetwork := range hsnnetworks {
+		if hnsnetwork.Name == networkName {
+			return &hnsnetwork, nil
+		}
+	}
+	return nil, fmt.Errorf("Network %v not found", networkName)
+}
+
+// Create Endpoint by sending EndpointRequest to HNS. TODO: Create a separate HNS interface to place all these methods
+func (endpoint *HNSEndpoint) Create() (*HNSEndpoint, error) {
+	jsonString, err := json.Marshal(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return HNSEndpointRequest("POST", "", string(jsonString))
+}
+
+// Create Endpoint by sending EndpointRequest to HNS
+func (endpoint *HNSEndpoint) Delete() (*HNSEndpoint, error) {
+	return HNSEndpointRequest("DELETE", endpoint.Id, "")
+}
+
+// GetHNSEndpointByID
+func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) {
+	return HNSEndpointRequest("GET", endpointID, "")
+}
+
+// GetHNSNetworkName filtered by Name
+func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) {
+	hnsResponse, err := HNSListEndpointRequest()
+	if err != nil {
+		return nil, err
+	}
+	for _, hnsEndpoint := range hnsResponse {
+		if hnsEndpoint.Name == endpointName {
+			return &hnsEndpoint, nil
+		}
+	}
+	return nil, fmt.Errorf("Endpoint %v not found", endpointName)
+}

vendor/github.com/Microsoft/hcsshim/importlayer.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/Microsoft/go-winio"
 	"github.com/Sirupsen/logrus"
@@ -209,6 +208,5 @@ func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string)
 	if err != nil {
 		return nil, makeError(err, "ImportLayerStart", "")
 	}
-	runtime.SetFinalizer(w, func(w *FilterLayerWriter) { w.Close() })
 	return w, nil
 }

vendor/github.com/Microsoft/hcsshim/interface.go

@@ -117,6 +117,9 @@ type Container interface {
 
 	// Close cleans up any state associated with the container but does not terminate or wait for it.
 	Close() error
+
+	// Modify the System
+	Modify(config *ResourceModificationRequestResponse) error
 }
 
 // Process represents a running or exited process.

vendor/github.com/Microsoft/hcsshim/legacy.go

@@ -23,6 +23,13 @@ var mutatedUtilityVMFiles = map[string]bool{
 	`EFI\Microsoft\Boot\BCD.LOG2`: true,
 }
 
+const (
+	filesPath          = `Files`
+	hivesPath          = `Hives`
+	utilityVMPath      = `UtilityVM`
+	utilityVMFilesPath = `UtilityVM\Files`
+)
+
 func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) {
 	return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition)
 }
@@ -44,6 +51,10 @@ func makeLongAbsPath(path string) (string, error) {
 	return `\\?\` + path, nil
 }
 
+func hasPathPrefix(p, prefix string) bool {
+	return strings.HasPrefix(p, prefix) && len(p) > len(prefix) && p[len(prefix)] == '\\'
+}
+
 type fileEntry struct {
 	path string
 	fi   os.FileInfo
@@ -83,7 +94,7 @@ func readTombstones(path string) (map[string]([]string), error) {
 
 	ts := make(map[string]([]string))
 	for s.Scan() {
-		t := filepath.Join("Files", s.Text()[1:]) // skip leading `\`
+		t := filepath.Join(filesPath, s.Text()[1:]) // skip leading `\`
 		dir := filepath.Dir(t)
 		ts[dir] = append(ts[dir], t)
 	}
@@ -212,7 +223,7 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 		return
 	}
 
-	if fe.fi.IsDir() && strings.HasPrefix(path, `Files\`) {
+	if fe.fi.IsDir() && hasPathPrefix(path, filesPath) {
 		fe.path += ".$wcidirs$"
 	}
 
@@ -231,14 +242,14 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 		return
 	}
 
-	if !strings.HasPrefix(path, `Files\`) {
+	if !hasPathPrefix(path, filesPath) {
 		size = fe.fi.Size()
 		r.backupReader = winio.NewBackupFileReader(f, false)
-		if path == "Hives" || path == "Files" {
+		if path == hivesPath || path == filesPath {
 			// The Hives directory has a non-deterministic file time because of the
 			// nature of the import process. Use the times from System_Delta.
 			var g *os.File
-			g, err = os.Open(filepath.Join(r.root, `Hives\System_Delta`))
+			g, err = os.Open(filepath.Join(r.root, hivesPath, `System_Delta`))
 			if err != nil {
 				return
 			}
@@ -357,7 +368,7 @@ func (w *legacyLayerWriter) init() error {
 
 func (w *legacyLayerWriter) initUtilityVM() error {
 	if !w.HasUtilityVM {
-		err := os.Mkdir(filepath.Join(w.destRoot, `UtilityVM`), 0)
+		err := os.Mkdir(filepath.Join(w.destRoot, utilityVMPath), 0)
 		if err != nil {
 			return err
 		}
@@ -365,7 +376,7 @@ func (w *legacyLayerWriter) initUtilityVM() error {
 		// clone the utility VM from the parent layer into this layer. Use hard
 		// links to avoid unnecessary copying, since most of the files are
 		// immutable.
-		err = cloneTree(filepath.Join(w.parentRoots[0], `UtilityVM\Files`), filepath.Join(w.destRoot, `UtilityVM\Files`), mutatedUtilityVMFiles)
+		err = cloneTree(filepath.Join(w.parentRoots[0], utilityVMFilesPath), filepath.Join(w.destRoot, utilityVMFilesPath), mutatedUtilityVMFiles)
 		if err != nil {
 			return fmt.Errorf("cloning the parent utility VM image failed: %s", err)
 		}
@@ -490,15 +501,15 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		return err
 	}
 
-	if name == `UtilityVM` {
+	if name == utilityVMPath {
 		return w.initUtilityVM()
 	}
 
-	if strings.HasPrefix(name, `UtilityVM\`) {
+	if hasPathPrefix(name, utilityVMPath) {
 		if !w.HasUtilityVM {
 			return errors.New("missing UtilityVM directory")
 		}
-		if !strings.HasPrefix(name, `UtilityVM\Files\`) && name != `UtilityVM\Files` {
+		if !hasPathPrefix(name, utilityVMFilesPath) && name != utilityVMFilesPath {
 			return errors.New("invalid UtilityVM layer")
 		}
 		path := filepath.Join(w.destRoot, name)
@@ -585,7 +596,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro
 		return err
 	}
 
-	if strings.HasPrefix(name, `Hives\`) {
+	if hasPathPrefix(name, hivesPath) {
 		w.backupWriter = winio.NewBackupFileWriter(f, false)
 	} else {
 		// The file attributes are written before the stream.
@@ -608,22 +619,19 @@ func (w *legacyLayerWriter) AddLink(name string, target string) error {
 		return err
 	}
 
-	var requiredPrefix string
 	var roots []string
-	if prefix := `Files\`; strings.HasPrefix(name, prefix) {
-		requiredPrefix = prefix
+	if hasPathPrefix(target, filesPath) {
 		// Look for cross-layer hard link targets in the parent layers, since
 		// nothing is in the destination path yet.
 		roots = w.parentRoots
-	} else if prefix := `UtilityVM\Files\`; strings.HasPrefix(name, prefix) {
-		requiredPrefix = prefix
+	} else if hasPathPrefix(target, utilityVMFilesPath) {
 		// Since the utility VM is fully cloned into the destination path
 		// already, look for cross-layer hard link targets directly in the
 		// destination path.
 		roots = []string{w.destRoot}
 	}
 
-	if requiredPrefix == "" || !strings.HasPrefix(target, requiredPrefix) {
+	if roots == nil || (!hasPathPrefix(name, filesPath) && !hasPathPrefix(name, utilityVMFilesPath)) {
 		return errors.New("invalid hard link in layer")
 	}
 
@@ -657,9 +665,9 @@ func (w *legacyLayerWriter) AddLink(name string, target string) error {
 }
 
 func (w *legacyLayerWriter) Remove(name string) error {
-	if strings.HasPrefix(name, `Files\`) {
-		w.tombstones = append(w.tombstones, name[len(`Files\`):])
-	} else if strings.HasPrefix(name, `UtilityVM\Files\`) {
+	if hasPathPrefix(name, filesPath) {
+		w.tombstones = append(w.tombstones, name[len(filesPath)+1:])
+	} else if hasPathPrefix(name, utilityVMFilesPath) {
 		err := w.initUtilityVM()
 		if err != nil {
 			return err

vendor/github.com/Microsoft/hcsshim/process.go

@@ -3,7 +3,6 @@ package hcsshim
 import (
 	"encoding/json"
 	"io"
-	"runtime"
 	"sync"
 	"syscall"
 	"time"
@@ -322,17 +321,11 @@ func (process *process) Close() error {
 	}
 
 	process.handle = 0
-	runtime.SetFinalizer(process, nil)
 
 	logrus.Debugf(title+" succeeded processid=%d", process.processID)
 	return nil
 }
 
-// closeProcess wraps process.Close for use by a finalizer
-func closeProcess(process *process) {
-	process.Close()
-}
-
 func (process *process) registerCallback() error {
 	context := &notifcationWatcherContext{
 		channels: newChannels(),