|
@@ -2,7 +2,6 @@ package hcs
|
|
|
|
|
|
import (
|
|
import (
|
|
"encoding/json"
|
|
"encoding/json"
|
|
- "fmt"
|
|
|
|
"os"
|
|
"os"
|
|
"strconv"
|
|
"strconv"
|
|
"sync"
|
|
"sync"
|
|
@@ -10,6 +9,7 @@ import (
|
|
"time"
|
|
"time"
|
|
|
|
|
|
"github.com/Microsoft/hcsshim/internal/interop"
|
|
"github.com/Microsoft/hcsshim/internal/interop"
|
|
|
|
+ "github.com/Microsoft/hcsshim/internal/logfields"
|
|
"github.com/Microsoft/hcsshim/internal/schema1"
|
|
"github.com/Microsoft/hcsshim/internal/schema1"
|
|
"github.com/Microsoft/hcsshim/internal/timeout"
|
|
"github.com/Microsoft/hcsshim/internal/timeout"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/sirupsen/logrus"
|
|
@@ -41,36 +41,74 @@ type System struct {
|
|
handle hcsSystem
|
|
handle hcsSystem
|
|
id string
|
|
id string
|
|
callbackNumber uintptr
|
|
callbackNumber uintptr
|
|
-}
|
|
|
|
|
|
|
|
-// CreateComputeSystem creates a new compute system with the given configuration but does not start it.
|
|
|
|
-func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System, error) {
|
|
|
|
- operation := "CreateComputeSystem"
|
|
|
|
- title := "hcsshim::" + operation
|
|
|
|
|
|
+ logctx logrus.Fields
|
|
|
|
+
|
|
|
|
+ closedWaitOnce sync.Once
|
|
|
|
+ waitBlock chan struct{}
|
|
|
|
+ waitError error
|
|
|
|
+}
|
|
|
|
|
|
- computeSystem := &System{
|
|
|
|
|
|
+func newSystem(id string) *System {
|
|
|
|
+ return &System{
|
|
id: id,
|
|
id: id,
|
|
|
|
+ logctx: logrus.Fields{
|
|
|
|
+ logfields.ContainerID: id,
|
|
|
|
+ },
|
|
|
|
+ waitBlock: make(chan struct{}),
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func (computeSystem *System) logOperationBegin(operation string) {
|
|
|
|
+ logOperationBegin(
|
|
|
|
+ computeSystem.logctx,
|
|
|
|
+ operation+" - Begin Operation")
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func (computeSystem *System) logOperationEnd(operation string, err error) {
|
|
|
|
+ var result string
|
|
|
|
+ if err == nil {
|
|
|
|
+ result = "Success"
|
|
|
|
+ } else {
|
|
|
|
+ result = "Error"
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ logOperationEnd(
|
|
|
|
+ computeSystem.logctx,
|
|
|
|
+ operation+" - End Operation - "+result,
|
|
|
|
+ err)
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// CreateComputeSystem creates a new compute system with the given configuration but does not start it.
|
|
|
|
+func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (_ *System, err error) {
|
|
|
|
+ operation := "hcsshim::CreateComputeSystem"
|
|
|
|
+
|
|
|
|
+ computeSystem := newSystem(id)
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
+
|
|
hcsDocumentB, err := json.Marshal(hcsDocumentInterface)
|
|
hcsDocumentB, err := json.Marshal(hcsDocumentInterface)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, err
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
|
|
hcsDocument := string(hcsDocumentB)
|
|
hcsDocument := string(hcsDocumentB)
|
|
- logrus.Debugf(title+" ID=%s config=%s", id, hcsDocument)
|
|
|
|
|
|
+
|
|
|
|
+ logrus.WithFields(computeSystem.logctx).
|
|
|
|
+ WithField(logfields.JSON, hcsDocument).
|
|
|
|
+ Debug("HCS ComputeSystem Document")
|
|
|
|
|
|
var (
|
|
var (
|
|
- resultp *uint16
|
|
|
|
- identity syscall.Handle
|
|
|
|
|
|
+ resultp *uint16
|
|
|
|
+ identity syscall.Handle
|
|
|
|
+ createError error
|
|
)
|
|
)
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("CreateCompleteSystem %s: %s", id, hcsDocument), &completed)
|
|
|
|
- createError := hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ createError = hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp)
|
|
|
|
+ })
|
|
|
|
|
|
if createError == nil || IsPending(createError) {
|
|
if createError == nil || IsPending(createError) {
|
|
- if err := computeSystem.registerCallback(); err != nil {
|
|
|
|
|
|
+ if err = computeSystem.registerCallback(); err != nil {
|
|
// Terminate the compute system if it still exists. We're okay to
|
|
// Terminate the compute system if it still exists. We're okay to
|
|
// ignore a failure here.
|
|
// ignore a failure here.
|
|
computeSystem.Terminate()
|
|
computeSystem.Terminate()
|
|
@@ -88,25 +126,30 @@ func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (*System,
|
|
return nil, makeSystemError(computeSystem, operation, hcsDocument, err, events)
|
|
return nil, makeSystemError(computeSystem, operation, hcsDocument, err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title+" succeeded id=%s handle=%d", id, computeSystem.handle)
|
|
|
|
|
|
+ go computeSystem.waitBackground()
|
|
|
|
+
|
|
return computeSystem, nil
|
|
return computeSystem, nil
|
|
}
|
|
}
|
|
|
|
|
|
// OpenComputeSystem opens an existing compute system by ID.
|
|
// OpenComputeSystem opens an existing compute system by ID.
|
|
-func OpenComputeSystem(id string) (*System, error) {
|
|
|
|
- operation := "OpenComputeSystem"
|
|
|
|
- title := "hcsshim::" + operation
|
|
|
|
- logrus.Debugf(title+" ID=%s", id)
|
|
|
|
-
|
|
|
|
- computeSystem := &System{
|
|
|
|
- id: id,
|
|
|
|
- }
|
|
|
|
|
|
+func OpenComputeSystem(id string) (_ *System, err error) {
|
|
|
|
+ operation := "hcsshim::OpenComputeSystem"
|
|
|
|
+
|
|
|
|
+ computeSystem := newSystem(id)
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() {
|
|
|
|
+ if IsNotExist(err) {
|
|
|
|
+ computeSystem.logOperationEnd(operation, nil)
|
|
|
|
+ } else {
|
|
|
|
+ computeSystem.logOperationEnd(operation, err)
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
|
|
var (
|
|
var (
|
|
handle hcsSystem
|
|
handle hcsSystem
|
|
resultp *uint16
|
|
resultp *uint16
|
|
)
|
|
)
|
|
- err := hcsOpenComputeSystem(id, &handle, &resultp)
|
|
|
|
|
|
+ err = hcsOpenComputeSystem(id, &handle, &resultp)
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, makeSystemError(computeSystem, operation, "", err, events)
|
|
return nil, makeSystemError(computeSystem, operation, "", err, events)
|
|
@@ -114,18 +157,35 @@ func OpenComputeSystem(id string) (*System, error) {
|
|
|
|
|
|
computeSystem.handle = handle
|
|
computeSystem.handle = handle
|
|
|
|
|
|
- if err := computeSystem.registerCallback(); err != nil {
|
|
|
|
|
|
+ if err = computeSystem.registerCallback(); err != nil {
|
|
return nil, makeSystemError(computeSystem, operation, "", err, nil)
|
|
return nil, makeSystemError(computeSystem, operation, "", err, nil)
|
|
}
|
|
}
|
|
|
|
+ go computeSystem.waitBackground()
|
|
|
|
|
|
- logrus.Debugf(title+" succeeded id=%s handle=%d", id, handle)
|
|
|
|
return computeSystem, nil
|
|
return computeSystem, nil
|
|
}
|
|
}
|
|
|
|
|
|
// GetComputeSystems gets a list of the compute systems on the system that match the query
|
|
// GetComputeSystems gets a list of the compute systems on the system that match the query
|
|
-func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerProperties, error) {
|
|
|
|
- operation := "GetComputeSystems"
|
|
|
|
- title := "hcsshim::" + operation
|
|
|
|
|
|
+func GetComputeSystems(q schema1.ComputeSystemQuery) (_ []schema1.ContainerProperties, err error) {
|
|
|
|
+ operation := "hcsshim::GetComputeSystems"
|
|
|
|
+ fields := logrus.Fields{}
|
|
|
|
+ logOperationBegin(
|
|
|
|
+ fields,
|
|
|
|
+ operation+" - Begin Operation")
|
|
|
|
+
|
|
|
|
+ defer func() {
|
|
|
|
+ var result string
|
|
|
|
+ if err == nil {
|
|
|
|
+ result = "Success"
|
|
|
|
+ } else {
|
|
|
|
+ result = "Error"
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ logOperationEnd(
|
|
|
|
+ fields,
|
|
|
|
+ operation+" - End Operation - "+result,
|
|
|
|
+ err)
|
|
|
|
+ }()
|
|
|
|
|
|
queryb, err := json.Marshal(q)
|
|
queryb, err := json.Marshal(q)
|
|
if err != nil {
|
|
if err != nil {
|
|
@@ -133,16 +193,19 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerPropert
|
|
}
|
|
}
|
|
|
|
|
|
query := string(queryb)
|
|
query := string(queryb)
|
|
- logrus.Debugf(title+" query=%s", query)
|
|
|
|
|
|
+
|
|
|
|
+ logrus.WithFields(fields).
|
|
|
|
+ WithField(logfields.JSON, query).
|
|
|
|
+ Debug("HCS ComputeSystem Query")
|
|
|
|
|
|
var (
|
|
var (
|
|
resultp *uint16
|
|
resultp *uint16
|
|
computeSystemsp *uint16
|
|
computeSystemsp *uint16
|
|
)
|
|
)
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("GetComputeSystems %s:", query), &completed)
|
|
|
|
- err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+
|
|
|
|
+ syscallWatcher(fields, func() {
|
|
|
|
+ err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, &HcsError{Op: operation, Err: err, Events: events}
|
|
return nil, &HcsError{Op: operation, Err: err, Events: events}
|
|
@@ -153,20 +216,21 @@ func GetComputeSystems(q schema1.ComputeSystemQuery) ([]schema1.ContainerPropert
|
|
}
|
|
}
|
|
computeSystemsRaw := interop.ConvertAndFreeCoTaskMemBytes(computeSystemsp)
|
|
computeSystemsRaw := interop.ConvertAndFreeCoTaskMemBytes(computeSystemsp)
|
|
computeSystems := []schema1.ContainerProperties{}
|
|
computeSystems := []schema1.ContainerProperties{}
|
|
- if err := json.Unmarshal(computeSystemsRaw, &computeSystems); err != nil {
|
|
|
|
|
|
+ if err = json.Unmarshal(computeSystemsRaw, &computeSystems); err != nil {
|
|
return nil, err
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return computeSystems, nil
|
|
return computeSystems, nil
|
|
}
|
|
}
|
|
|
|
|
|
// Start synchronously starts the computeSystem.
|
|
// Start synchronously starts the computeSystem.
|
|
-func (computeSystem *System) Start() error {
|
|
|
|
|
|
+func (computeSystem *System) Start() (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::Start ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Start"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Start", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Start", "", ErrAlreadyClosed, nil)
|
|
@@ -199,16 +263,14 @@ func (computeSystem *System) Start() error {
|
|
}
|
|
}
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("StartComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsStartComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsStartComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
+ })
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart)
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart)
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Start", "", err, events)
|
|
return makeSystemError(computeSystem, "Start", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
@@ -219,98 +281,152 @@ func (computeSystem *System) ID() string {
|
|
|
|
|
|
// Shutdown requests a compute system shutdown, if IsPending() on the error returned is true,
|
|
// Shutdown requests a compute system shutdown, if IsPending() on the error returned is true,
|
|
// it may not actually be shut down until Wait() succeeds.
|
|
// it may not actually be shut down until Wait() succeeds.
|
|
-func (computeSystem *System) Shutdown() error {
|
|
|
|
|
|
+func (computeSystem *System) Shutdown() (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::Shutdown"
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Shutdown"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() {
|
|
|
|
+ if IsAlreadyClosed(err) || IsAlreadyStopped(err) || IsPending(err) {
|
|
|
|
+ computeSystem.logOperationEnd(operation, nil)
|
|
|
|
+ } else {
|
|
|
|
+ computeSystem.logOperationEnd(operation, err)
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
+
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Shutdown", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Shutdown", "", ErrAlreadyClosed, nil)
|
|
}
|
|
}
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("ShutdownComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsShutdownComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsShutdownComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Shutdown", "", err, events)
|
|
return makeSystemError(computeSystem, "Shutdown", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
// Terminate requests a compute system terminate, if IsPending() on the error returned is true,
|
|
// Terminate requests a compute system terminate, if IsPending() on the error returned is true,
|
|
// it may not actually be shut down until Wait() succeeds.
|
|
// it may not actually be shut down until Wait() succeeds.
|
|
-func (computeSystem *System) Terminate() error {
|
|
|
|
|
|
+func (computeSystem *System) Terminate() (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::Terminate ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Terminate"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() {
|
|
|
|
+ if IsAlreadyClosed(err) || IsAlreadyStopped(err) || IsPending(err) {
|
|
|
|
+ computeSystem.logOperationEnd(operation, nil)
|
|
|
|
+ } else {
|
|
|
|
+ computeSystem.logOperationEnd(operation, err)
|
|
|
|
+ }
|
|
|
|
+ }()
|
|
|
|
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Terminate", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Terminate", "", ErrAlreadyClosed, nil)
|
|
}
|
|
}
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("TerminateComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsTerminateComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsTerminateComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
- if err != nil {
|
|
|
|
|
|
+ if err != nil && err != ErrVmcomputeAlreadyStopped {
|
|
return makeSystemError(computeSystem, "Terminate", "", err, events)
|
|
return makeSystemError(computeSystem, "Terminate", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-// Wait synchronously waits for the compute system to shutdown or terminate.
|
|
|
|
-func (computeSystem *System) Wait() error {
|
|
|
|
- title := "hcsshim::ComputeSystem::Wait ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+// waitBackground waits for the compute system exit notification. Once received
|
|
|
|
+// sets `computeSystem.waitError` (if any) and unblocks all `Wait`,
|
|
|
|
+// `WaitExpectedError`, and `WaitTimeout` calls.
|
|
|
|
+//
|
|
|
|
+// This MUST be called exactly once per `computeSystem.handle` but `Wait`,
|
|
|
|
+// `WaitExpectedError`, and `WaitTimeout` are safe to call multiple times.
|
|
|
|
+func (computeSystem *System) waitBackground() {
|
|
|
|
+ computeSystem.waitError = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil)
|
|
|
|
+ computeSystem.closedWaitOnce.Do(func() {
|
|
|
|
+ close(computeSystem.waitBlock)
|
|
|
|
+ })
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// Wait synchronously waits for the compute system to shutdown or terminate. If
|
|
|
|
+// the compute system has already exited returns the previous error (if any).
|
|
|
|
+func (computeSystem *System) Wait() (err error) {
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Wait"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
- err := waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil)
|
|
|
|
- if err != nil {
|
|
|
|
- return makeSystemError(computeSystem, "Wait", "", err, nil)
|
|
|
|
|
|
+ <-computeSystem.waitBlock
|
|
|
|
+ if computeSystem.waitError != nil {
|
|
|
|
+ return makeSystemError(computeSystem, "Wait", "", computeSystem.waitError, nil)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-// WaitTimeout synchronously waits for the compute system to terminate or the duration to elapse.
|
|
|
|
-// If the timeout expires, IsTimeout(err) == true
|
|
|
|
-func (computeSystem *System) WaitTimeout(timeout time.Duration) error {
|
|
|
|
- title := "hcsshim::ComputeSystem::WaitTimeout ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+// WaitExpectedError synchronously waits for the compute system to shutdown or
|
|
|
|
+// terminate and returns the error (if any) as long as it does not match
|
|
|
|
+// `expected`. If the compute system has already exited returns the previous
|
|
|
|
+// error (if any) as long as it does not match `expected`.
|
|
|
|
+func (computeSystem *System) WaitExpectedError(expected error) (err error) {
|
|
|
|
+ operation := "hcsshim::ComputeSystem::WaitExpectedError"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
- err := waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, &timeout)
|
|
|
|
- if err != nil {
|
|
|
|
- return makeSystemError(computeSystem, "WaitTimeout", "", err, nil)
|
|
|
|
|
|
+ <-computeSystem.waitBlock
|
|
|
|
+ if computeSystem.waitError != nil && getInnerError(computeSystem.waitError) != expected {
|
|
|
|
+ return makeSystemError(computeSystem, "WaitExpectedError", "", computeSystem.waitError, nil)
|
|
}
|
|
}
|
|
-
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1.ContainerProperties, error) {
|
|
|
|
|
|
+// WaitTimeout synchronously waits for the compute system to terminate or the
|
|
|
|
+// duration to elapse. If the timeout expires, `IsTimeout(err) == true`. If
|
|
|
|
+// the compute system has already exited returns the previous error (if any).
|
|
|
|
+func (computeSystem *System) WaitTimeout(timeout time.Duration) (err error) {
|
|
|
|
+ operation := "hcsshim::ComputeSystem::WaitTimeout"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
+
|
|
|
|
+ select {
|
|
|
|
+ case <-computeSystem.waitBlock:
|
|
|
|
+ if computeSystem.waitError != nil {
|
|
|
|
+ return makeSystemError(computeSystem, "WaitTimeout", "", computeSystem.waitError, nil)
|
|
|
|
+ }
|
|
|
|
+ return nil
|
|
|
|
+ case <-time.After(timeout):
|
|
|
|
+ return makeSystemError(computeSystem, "WaitTimeout", "", ErrTimeout, nil)
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func (computeSystem *System) Properties(types ...schema1.PropertyType) (_ *schema1.ContainerProperties, err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
|
|
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Properties"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
+
|
|
queryj, err := json.Marshal(schema1.PropertyQuery{types})
|
|
queryj, err := json.Marshal(schema1.PropertyQuery{types})
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ logrus.WithFields(computeSystem.logctx).
|
|
|
|
+ WithField(logfields.JSON, queryj).
|
|
|
|
+ Debug("HCS ComputeSystem Properties Query")
|
|
|
|
+
|
|
var resultp, propertiesp *uint16
|
|
var resultp, propertiesp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("GetComputeSystemProperties %s:", computeSystem.ID()), &completed)
|
|
|
|
- err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, events)
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, events)
|
|
@@ -324,64 +440,69 @@ func (computeSystem *System) Properties(types ...schema1.PropertyType) (*schema1
|
|
if err := json.Unmarshal(propertiesRaw, properties); err != nil {
|
|
if err := json.Unmarshal(propertiesRaw, properties); err != nil {
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
|
|
return nil, makeSystemError(computeSystem, "Properties", "", err, nil)
|
|
}
|
|
}
|
|
|
|
+
|
|
return properties, nil
|
|
return properties, nil
|
|
}
|
|
}
|
|
|
|
|
|
// Pause pauses the execution of the computeSystem. This feature is not enabled in TP5.
|
|
// Pause pauses the execution of the computeSystem. This feature is not enabled in TP5.
|
|
-func (computeSystem *System) Pause() error {
|
|
|
|
|
|
+func (computeSystem *System) Pause() (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::Pause ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Pause"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Pause", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Pause", "", ErrAlreadyClosed, nil)
|
|
}
|
|
}
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("PauseComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsPauseComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsPauseComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
+ })
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Pause", "", err, events)
|
|
return makeSystemError(computeSystem, "Pause", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
// Resume resumes the execution of the computeSystem. This feature is not enabled in TP5.
|
|
// Resume resumes the execution of the computeSystem. This feature is not enabled in TP5.
|
|
-func (computeSystem *System) Resume() error {
|
|
|
|
|
|
+func (computeSystem *System) Resume() (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::Resume ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Resume"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Resume", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Resume", "", ErrAlreadyClosed, nil)
|
|
}
|
|
}
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("ResumeComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsResumeComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsResumeComputeSystem(computeSystem.handle, "", &resultp)
|
|
|
|
+ })
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
|
|
events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Resume", "", err, events)
|
|
return makeSystemError(computeSystem, "Resume", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
// CreateProcess launches a new process within the computeSystem.
|
|
// CreateProcess launches a new process within the computeSystem.
|
|
-func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) {
|
|
|
|
|
|
+func (computeSystem *System) CreateProcess(c interface{}) (_ *Process, err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::CreateProcess ID=" + computeSystem.ID()
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::CreateProcess"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
+
|
|
var (
|
|
var (
|
|
processInfo hcsProcessInformation
|
|
processInfo hcsProcessInformation
|
|
processHandle hcsProcess
|
|
processHandle hcsProcess
|
|
@@ -398,42 +519,51 @@ func (computeSystem *System) CreateProcess(c interface{}) (*Process, error) {
|
|
}
|
|
}
|
|
|
|
|
|
configuration := string(configurationb)
|
|
configuration := string(configurationb)
|
|
- logrus.Debugf(title+" config=%s", configuration)
|
|
|
|
|
|
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("CreateProcess %s: %s", computeSystem.ID(), configuration), &completed)
|
|
|
|
- err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ logrus.WithFields(computeSystem.logctx).
|
|
|
|
+ WithField(logfields.JSON, configuration).
|
|
|
|
+ Debug("HCS ComputeSystem Process Document")
|
|
|
|
+
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events)
|
|
return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- process := &Process{
|
|
|
|
- handle: processHandle,
|
|
|
|
- processID: int(processInfo.ProcessId),
|
|
|
|
- system: computeSystem,
|
|
|
|
- cachedPipes: &cachedPipes{
|
|
|
|
- stdIn: processInfo.StdInput,
|
|
|
|
- stdOut: processInfo.StdOutput,
|
|
|
|
- stdErr: processInfo.StdError,
|
|
|
|
- },
|
|
|
|
|
|
+ logrus.WithFields(computeSystem.logctx).
|
|
|
|
+ WithField(logfields.ProcessID, processInfo.ProcessId).
|
|
|
|
+ Debug("HCS ComputeSystem CreateProcess PID")
|
|
|
|
+
|
|
|
|
+ process := newProcess(processHandle, int(processInfo.ProcessId), computeSystem)
|
|
|
|
+ process.cachedPipes = &cachedPipes{
|
|
|
|
+ stdIn: processInfo.StdInput,
|
|
|
|
+ stdOut: processInfo.StdOutput,
|
|
|
|
+ stdErr: processInfo.StdError,
|
|
}
|
|
}
|
|
|
|
|
|
- if err := process.registerCallback(); err != nil {
|
|
|
|
|
|
+ if err = process.registerCallback(); err != nil {
|
|
return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil)
|
|
return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil)
|
|
}
|
|
}
|
|
|
|
+ go process.waitBackground()
|
|
|
|
|
|
- logrus.Debugf(title+" succeeded processid=%d", process.processID)
|
|
|
|
return process, nil
|
|
return process, nil
|
|
}
|
|
}
|
|
|
|
|
|
// OpenProcess gets an interface to an existing process within the computeSystem.
|
|
// OpenProcess gets an interface to an existing process within the computeSystem.
|
|
-func (computeSystem *System) OpenProcess(pid int) (*Process, error) {
|
|
|
|
|
|
+func (computeSystem *System) OpenProcess(pid int) (_ *Process, err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::ComputeSystem::OpenProcess ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title+" processid=%d", pid)
|
|
|
|
|
|
+
|
|
|
|
+ // Add PID for the context of this operation
|
|
|
|
+ computeSystem.logctx[logfields.ProcessID] = pid
|
|
|
|
+ defer delete(computeSystem.logctx, logfields.ProcessID)
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::OpenProcess"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
+
|
|
var (
|
|
var (
|
|
processHandle hcsProcess
|
|
processHandle hcsProcess
|
|
resultp *uint16
|
|
resultp *uint16
|
|
@@ -443,56 +573,53 @@ func (computeSystem *System) OpenProcess(pid int) (*Process, error) {
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil)
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil)
|
|
}
|
|
}
|
|
|
|
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("OpenProcess %s: %d", computeSystem.ID(), pid), &completed)
|
|
|
|
- err := hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events)
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events)
|
|
}
|
|
}
|
|
|
|
|
|
- process := &Process{
|
|
|
|
- handle: processHandle,
|
|
|
|
- processID: pid,
|
|
|
|
- system: computeSystem,
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- if err := process.registerCallback(); err != nil {
|
|
|
|
|
|
+ process := newProcess(processHandle, pid, computeSystem)
|
|
|
|
+ if err = process.registerCallback(); err != nil {
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", err, nil)
|
|
return nil, makeSystemError(computeSystem, "OpenProcess", "", err, nil)
|
|
}
|
|
}
|
|
|
|
+ go process.waitBackground()
|
|
|
|
|
|
- logrus.Debugf(title+" succeeded processid=%s", process.processID)
|
|
|
|
return process, nil
|
|
return process, nil
|
|
}
|
|
}
|
|
|
|
|
|
// Close cleans up any state associated with the compute system but does not terminate or wait for it.
|
|
// Close cleans up any state associated with the compute system but does not terminate or wait for it.
|
|
-func (computeSystem *System) Close() error {
|
|
|
|
|
|
+func (computeSystem *System) Close() (err error) {
|
|
computeSystem.handleLock.Lock()
|
|
computeSystem.handleLock.Lock()
|
|
defer computeSystem.handleLock.Unlock()
|
|
defer computeSystem.handleLock.Unlock()
|
|
- title := "hcsshim::ComputeSystem::Close ID=" + computeSystem.ID()
|
|
|
|
- logrus.Debugf(title)
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Close"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
// Don't double free this
|
|
// Don't double free this
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
- if err := computeSystem.unregisterCallback(); err != nil {
|
|
|
|
|
|
+ if err = computeSystem.unregisterCallback(); err != nil {
|
|
return makeSystemError(computeSystem, "Close", "", err, nil)
|
|
return makeSystemError(computeSystem, "Close", "", err, nil)
|
|
}
|
|
}
|
|
|
|
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("CloseComputeSystem %s:", computeSystem.ID()), &completed)
|
|
|
|
- err := hcsCloseComputeSystem(computeSystem.handle)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsCloseComputeSystem(computeSystem.handle)
|
|
|
|
+ })
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Close", "", err, nil)
|
|
return makeSystemError(computeSystem, "Close", "", err, nil)
|
|
}
|
|
}
|
|
|
|
|
|
computeSystem.handle = 0
|
|
computeSystem.handle = 0
|
|
|
|
+ computeSystem.closedWaitOnce.Do(func() {
|
|
|
|
+ close(computeSystem.waitBlock)
|
|
|
|
+ })
|
|
|
|
|
|
- logrus.Debugf(title + " succeeded")
|
|
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
@@ -545,7 +672,7 @@ func (computeSystem *System) unregisterCallback() error {
|
|
closeChannels(context.channels)
|
|
closeChannels(context.channels)
|
|
|
|
|
|
callbackMapLock.Lock()
|
|
callbackMapLock.Lock()
|
|
- callbackMap[callbackNumber] = nil
|
|
|
|
|
|
+ delete(callbackMap, callbackNumber)
|
|
callbackMapLock.Unlock()
|
|
callbackMapLock.Unlock()
|
|
|
|
|
|
handle = 0
|
|
handle = 0
|
|
@@ -553,11 +680,14 @@ func (computeSystem *System) unregisterCallback() error {
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
|
|
-// Modifies the System by sending a request to HCS
|
|
|
|
-func (computeSystem *System) Modify(config interface{}) error {
|
|
|
|
|
|
+// Modify the System by sending a request to HCS
|
|
|
|
+func (computeSystem *System) Modify(config interface{}) (err error) {
|
|
computeSystem.handleLock.RLock()
|
|
computeSystem.handleLock.RLock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
defer computeSystem.handleLock.RUnlock()
|
|
- title := "hcsshim::Modify ID=" + computeSystem.id
|
|
|
|
|
|
+
|
|
|
|
+ operation := "hcsshim::ComputeSystem::Modify"
|
|
|
|
+ computeSystem.logOperationBegin(operation)
|
|
|
|
+ defer func() { computeSystem.logOperationEnd(operation, err) }()
|
|
|
|
|
|
if computeSystem.handle == 0 {
|
|
if computeSystem.handle == 0 {
|
|
return makeSystemError(computeSystem, "Modify", "", ErrAlreadyClosed, nil)
|
|
return makeSystemError(computeSystem, "Modify", "", ErrAlreadyClosed, nil)
|
|
@@ -569,17 +699,19 @@ func (computeSystem *System) Modify(config interface{}) error {
|
|
}
|
|
}
|
|
|
|
|
|
requestString := string(requestJSON)
|
|
requestString := string(requestJSON)
|
|
- logrus.Debugf(title + " " + requestString)
|
|
|
|
|
|
+
|
|
|
|
+ logrus.WithFields(computeSystem.logctx).
|
|
|
|
+ WithField(logfields.JSON, requestString).
|
|
|
|
+ Debug("HCS ComputeSystem Modify Document")
|
|
|
|
|
|
var resultp *uint16
|
|
var resultp *uint16
|
|
- completed := false
|
|
|
|
- go syscallWatcher(fmt.Sprintf("ModifyComputeSystem %s: %s", computeSystem.ID(), requestString), &completed)
|
|
|
|
- err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp)
|
|
|
|
- completed = true
|
|
|
|
|
|
+ syscallWatcher(computeSystem.logctx, func() {
|
|
|
|
+ err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp)
|
|
|
|
+ })
|
|
events := processHcsResult(resultp)
|
|
events := processHcsResult(resultp)
|
|
if err != nil {
|
|
if err != nil {
|
|
return makeSystemError(computeSystem, "Modify", requestString, err, events)
|
|
return makeSystemError(computeSystem, "Modify", requestString, err, events)
|
|
}
|
|
}
|
|
- logrus.Debugf(title + " succeeded ")
|
|
|
|
|
|
+
|
|
return nil
|
|
return nil
|
|
}
|
|
}
|