Merge pull request #43915 from thaJeztah/22.06_backport_vendor_hcsshim

[22.06 backport] vendor: github.com/Microsoft/hcsshim v0.9.4

vendor.mod

@@ -12,7 +12,7 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1
 	github.com/Graylog2/go-gelf v0.0.0-20191017102106-1550ee647df0
 	github.com/Microsoft/go-winio v0.5.2
-	github.com/Microsoft/hcsshim v0.9.3
+	github.com/Microsoft/hcsshim v0.9.4
 	github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91
 	github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310
 	github.com/aws/aws-sdk-go v1.31.6

vendor.sum

@@ -94,8 +94,8 @@ github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwT
 github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg=
 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
-github.com/Microsoft/hcsshim v0.9.3 h1:k371PzBuRrz2b+ebGuI2nVgVhgsVX60jMfSw80NECxo=
-github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
+github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I=
+github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=

vendor/github.com/Microsoft/hcsshim/internal/jobobject/iocp.go

@@ -57,7 +57,7 @@ func pollIOCP(ctx context.Context, iocpHandle windows.Handle) {
 				}).Warn("failed to parse job object message")
 				continue
 			}
-			if err := msq.Write(notification); err == queue.ErrQueueClosed {
+			if err := msq.Enqueue(notification); err == queue.ErrQueueClosed {
 				// Write will only return an error when the queue is closed.
 				// The only time a queue would ever be closed is when we call `Close` on
 				// the job it belongs to which also removes it from the jobMap, so something

vendor/github.com/Microsoft/hcsshim/internal/jobobject/jobobject.go

@@ -68,6 +68,9 @@ type Options struct {
 	// `UseNTVariant` specifies if we should use the `Nt` variant of Open/CreateJobObject.
 	// Defaults to false.
 	UseNTVariant bool
+	// `IOTracking` enables tracking I/O statistics on the job object. More specifically this
+	// calls SetInformationJobObject with the JobObjectIoAttribution class.
+	EnableIOTracking bool
 }
 
 // Create creates a job object.
@@ -134,6 +137,12 @@ func Create(ctx context.Context, options *Options) (_ *JobObject, err error) {
 		job.mq = mq
 	}
 
+	if options.EnableIOTracking {
+		if err := enableIOTracking(jobHandle); err != nil {
+			return nil, err
+		}
+	}
+
 	return job, nil
 }
 
@@ -235,7 +244,7 @@ func (job *JobObject) PollNotification() (interface{}, error) {
 	if job.mq == nil {
 		return nil, ErrNotRegistered
 	}
-	return job.mq.ReadOrWait()
+	return job.mq.Dequeue()
 }
 
 // UpdateProcThreadAttribute updates the passed in ProcThreadAttributeList to contain what is necessary to
@@ -330,7 +339,7 @@ func (job *JobObject) Pids() ([]uint32, error) {
 	err := winapi.QueryInformationJobObject(
 		job.handle,
 		winapi.JobObjectBasicProcessIdList,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	)
@@ -356,7 +365,7 @@ func (job *JobObject) Pids() ([]uint32, error) {
 	if err = winapi.QueryInformationJobObject(
 		job.handle,
 		winapi.JobObjectBasicProcessIdList,
-		uintptr(unsafe.Pointer(&buf[0])),
+		unsafe.Pointer(&buf[0]),
 		uint32(len(buf)),
 		nil,
 	); err != nil {
@@ -384,7 +393,7 @@ func (job *JobObject) QueryMemoryStats() (*winapi.JOBOBJECT_MEMORY_USAGE_INFORMA
 	if err := winapi.QueryInformationJobObject(
 		job.handle,
 		winapi.JobObjectMemoryUsageInformation,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	); err != nil {
@@ -406,7 +415,7 @@ func (job *JobObject) QueryProcessorStats() (*winapi.JOBOBJECT_BASIC_ACCOUNTING_
 	if err := winapi.QueryInformationJobObject(
 		job.handle,
 		winapi.JobObjectBasicAccountingInformation,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	); err != nil {
@@ -415,7 +424,9 @@ func (job *JobObject) QueryProcessorStats() (*winapi.JOBOBJECT_BASIC_ACCOUNTING_
 	return &info, nil
 }
 
-// QueryStorageStats gets the storage (I/O) stats for the job object.
+// QueryStorageStats gets the storage (I/O) stats for the job object. This call will error
+// if either `EnableIOTracking` wasn't set to true on creation of the job, or SetIOTracking()
+// hasn't been called since creation of the job.
 func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_IO_ATTRIBUTION_INFORMATION, error) {
 	job.handleLock.RLock()
 	defer job.handleLock.RUnlock()
@@ -430,7 +441,7 @@ func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_IO_ATTRIBUTION_INFO
 	if err := winapi.QueryInformationJobObject(
 		job.handle,
 		winapi.JobObjectIoAttribution,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	); err != nil {
@@ -476,7 +487,7 @@ func (job *JobObject) QueryPrivateWorkingSet() (uint64, error) {
 		status := winapi.NtQueryInformationProcess(
 			h,
 			winapi.ProcessVmCounters,
-			uintptr(unsafe.Pointer(&vmCounters)),
+			unsafe.Pointer(&vmCounters),
 			uint32(unsafe.Sizeof(vmCounters)),
 			nil,
 		)
@@ -497,3 +508,31 @@ func (job *JobObject) QueryPrivateWorkingSet() (uint64, error) {
 
 	return jobWorkingSetSize, nil
 }
+
+// SetIOTracking enables IO tracking for processes in the job object.
+// This enables use of the QueryStorageStats method.
+func (job *JobObject) SetIOTracking() error {
+	job.handleLock.RLock()
+	defer job.handleLock.RUnlock()
+
+	if job.handle == 0 {
+		return ErrAlreadyClosed
+	}
+
+	return enableIOTracking(job.handle)
+}
+
+func enableIOTracking(job windows.Handle) error {
+	info := winapi.JOBOBJECT_IO_ATTRIBUTION_INFORMATION{
+		ControlFlags: winapi.JOBOBJECT_IO_ATTRIBUTION_CONTROL_ENABLE,
+	}
+	if _, err := windows.SetInformationJobObject(
+		job,
+		winapi.JobObjectIoAttribution,
+		uintptr(unsafe.Pointer(&info)),
+		uint32(unsafe.Sizeof(info)),
+	); err != nil {
+		return fmt.Errorf("failed to enable IO tracking on job object: %w", err)
+	}
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/jobobject/limits.go

@@ -202,7 +202,7 @@ func (job *JobObject) getExtendedInformation() (*windows.JOBOBJECT_EXTENDED_LIMI
 	if err := winapi.QueryInformationJobObject(
 		job.handle,
 		windows.JobObjectExtendedLimitInformation,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	); err != nil {
@@ -224,7 +224,7 @@ func (job *JobObject) getCPURateControlInformation() (*winapi.JOBOBJECT_CPU_RATE
 	if err := winapi.QueryInformationJobObject(
 		job.handle,
 		windows.JobObjectCpuRateControlInformation,
-		uintptr(unsafe.Pointer(&info)),
+		unsafe.Pointer(&info),
 		uint32(unsafe.Sizeof(info)),
 		nil,
 	); err != nil {

vendor/github.com/Microsoft/hcsshim/internal/queue/mq.go

@@ -5,10 +5,7 @@ import (
 	"sync"
 )
 
-var (
-	ErrQueueClosed = errors.New("the queue is closed for reading and writing")
-	ErrQueueEmpty  = errors.New("the queue is empty")
-)
+var ErrQueueClosed = errors.New("the queue is closed for reading and writing")
 
 // MessageQueue represents a threadsafe message queue to be used to retrieve or
 // write messages to.
@@ -29,8 +26,8 @@ func NewMessageQueue() *MessageQueue {
 	}
 }
 
-// Write writes `msg` to the queue.
-func (mq *MessageQueue) Write(msg interface{}) error {
+// Enqueue writes `msg` to the queue.
+func (mq *MessageQueue) Enqueue(msg interface{}) error {
 	mq.m.Lock()
 	defer mq.m.Unlock()
 
@@ -43,55 +40,37 @@ func (mq *MessageQueue) Write(msg interface{}) error {
 	return nil
 }
 
-// Read will read a value from the queue if available, otherwise return an error.
-func (mq *MessageQueue) Read() (interface{}, error) {
+// Dequeue will read a value from the queue and remove it. If the queue
+// is empty, this will block until the queue is closed or a value gets enqueued.
+func (mq *MessageQueue) Dequeue() (interface{}, error) {
 	mq.m.Lock()
 	defer mq.m.Unlock()
+
+	for !mq.closed && mq.size() == 0 {
+		mq.c.Wait()
+	}
+
+	// We got woken up, check if it's because the queue got closed.
 	if mq.closed {
 		return nil, ErrQueueClosed
 	}
-	if mq.isEmpty() {
-		return nil, ErrQueueEmpty
-	}
+
 	val := mq.messages[0]
 	mq.messages[0] = nil
 	mq.messages = mq.messages[1:]
 	return val, nil
 }
 
-// ReadOrWait will read a value from the queue if available, else it will wait for a
-// value to become available. This will block forever if nothing gets written or until
-// the queue gets closed.
-func (mq *MessageQueue) ReadOrWait() (interface{}, error) {
-	mq.m.Lock()
-	if mq.closed {
-		mq.m.Unlock()
-		return nil, ErrQueueClosed
-	}
-	if mq.isEmpty() {
-		for !mq.closed && mq.isEmpty() {
-			mq.c.Wait()
-		}
-		mq.m.Unlock()
-		return mq.Read()
-	}
-	val := mq.messages[0]
-	mq.messages[0] = nil
-	mq.messages = mq.messages[1:]
-	mq.m.Unlock()
-	return val, nil
-}
-
-// IsEmpty returns if the queue is empty
-func (mq *MessageQueue) IsEmpty() bool {
+// Size returns the size of the queue.
+func (mq *MessageQueue) Size() int {
 	mq.m.RLock()
 	defer mq.m.RUnlock()
-	return len(mq.messages) == 0
+	return mq.size()
 }
 
-// Nonexported empty check that doesn't lock so we can call this in Read and Write.
-func (mq *MessageQueue) isEmpty() bool {
-	return len(mq.messages) == 0
+// Nonexported size check to check if the queue is empty inside already locked functions.
+func (mq *MessageQueue) size() int {
+	return len(mq.messages)
 }
 
 // Close closes the queue for future writes or reads. Any attempts to read or write from the
@@ -99,13 +78,15 @@ func (mq *MessageQueue) isEmpty() bool {
 func (mq *MessageQueue) Close() {
 	mq.m.Lock()
 	defer mq.m.Unlock()
-	// Already closed
+
+	// Already closed, noop
 	if mq.closed {
 		return
 	}
+
 	mq.messages = nil
 	mq.closed = true
-	// If there's anybody currently waiting on a value from ReadOrWait, we need to
+	// If there's anybody currently waiting on a value from Dequeue, we need to
 	// broadcast so the read(s) can return ErrQueueClosed.
 	mq.c.Broadcast()
 }

vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go

@@ -175,7 +175,7 @@ type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct {
 //		LPDWORD            lpReturnLength
 // );
 //
-//sys QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo uintptr, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) = kernel32.QueryInformationJobObject
+//sys QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo unsafe.Pointer, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) = kernel32.QueryInformationJobObject
 
 // HANDLE OpenJobObjectW(
 //		DWORD   dwDesiredAccess,

vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go

@@ -18,7 +18,7 @@ const ProcessVmCounters = 3
 // 	[out, optional] PULONG           ReturnLength
 // );
 //
-//sys NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo uintptr, processInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQueryInformationProcess
+//sys NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo unsafe.Pointer, processInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQueryInformationProcess
 
 // typedef struct _VM_COUNTERS_EX
 // {

vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go

@@ -12,7 +12,8 @@ const STATUS_INFO_LENGTH_MISMATCH = 0xC0000004
 // 	ULONG                    SystemInformationLength,
 // 	PULONG                   ReturnLength
 // );
-//sys NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQuerySystemInformation
+//
+//sys NtQuerySystemInformation(systemInfoClass int, systemInformation unsafe.Pointer, systemInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQuerySystemInformation
 
 type SYSTEM_PROCESS_INFORMATION struct {
 	NextEntryOffset              uint32         // ULONG

vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go

@@ -100,7 +100,7 @@ func resizePseudoConsole(hPc windows.Handle, size uint32) (hr error) {
 	return
 }
 
-func NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) {
+func NtQuerySystemInformation(systemInfoClass int, systemInformation unsafe.Pointer, systemInfoLength uint32, returnLength *uint32) (status uint32) {
 	r0, _, _ := syscall.Syscall6(procNtQuerySystemInformation.Addr(), 4, uintptr(systemInfoClass), uintptr(systemInformation), uintptr(systemInfoLength), uintptr(unsafe.Pointer(returnLength)), 0, 0)
 	status = uint32(r0)
 	return
@@ -152,7 +152,7 @@ func IsProcessInJob(procHandle windows.Handle, jobHandle windows.Handle, result
 	return
 }
 
-func QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo uintptr, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) {
+func QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo unsafe.Pointer, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) {
 	r1, _, e1 := syscall.Syscall6(procQueryInformationJobObject.Addr(), 5, uintptr(jobHandle), uintptr(infoClass), uintptr(jobObjectInfo), uintptr(jobObjectInformationLength), uintptr(unsafe.Pointer(lpReturnLength)), 0)
 	if r1 == 0 {
 		if e1 != 0 {
@@ -244,7 +244,7 @@ func LocalFree(ptr uintptr) {
 	return
 }
 
-func NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo uintptr, processInfoLength uint32, returnLength *uint32) (status uint32) {
+func NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo unsafe.Pointer, processInfoLength uint32, returnLength *uint32) (status uint32) {
 	r0, _, _ := syscall.Syscall6(procNtQueryInformationProcess.Addr(), 5, uintptr(processHandle), uintptr(processInfoClass), uintptr(processInfo), uintptr(processInfoLength), uintptr(unsafe.Pointer(returnLength)), 0)
 	status = uint32(r0)
 	return

vendor/modules.txt

@@ -27,7 +27,7 @@ github.com/Microsoft/go-winio/pkg/etwlogrus
 github.com/Microsoft/go-winio/pkg/guid
 github.com/Microsoft/go-winio/pkg/security
 github.com/Microsoft/go-winio/vhd
-# github.com/Microsoft/hcsshim v0.9.3
+# github.com/Microsoft/hcsshim v0.9.4
 ## explicit; go 1.13
 github.com/Microsoft/hcsshim
 github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options