diff --git a/hack/vendor.sh b/hack/vendor.sh index 73a4458af5..0ddb204c6a 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -43,7 +43,7 @@ esac # the following lines are in sorted order, FYI clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62 -clone git github.com/Microsoft/hcsshim v0.3.0 +clone git github.com/Microsoft/hcsshim v0.3.1 clone git github.com/Microsoft/go-winio v0.3.4 clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 94c24c9e37..0695138519 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -6740,3 +6740,18 @@ func (s *DockerSuite) TestBuildLabelsOverride(c *check.C) { } } + +// Test case for #22855 +func (s *DockerSuite) TestBuildDeleteCommittedFile(c *check.C) { + name := "test-delete-committed-file" + + _, err := buildImage(name, + `FROM busybox + RUN echo test > file + RUN test -e file + RUN rm file + RUN sh -c "! test -e file"`, false) + if err != nil { + c.Fatal(err) + } +} diff --git a/vendor/src/github.com/Microsoft/hcsshim/callback.go b/vendor/src/github.com/Microsoft/hcsshim/callback.go index 122db1db18..6812384e14 100644 --- a/vendor/src/github.com/Microsoft/hcsshim/callback.go +++ b/vendor/src/github.com/Microsoft/hcsshim/callback.go @@ -40,39 +40,36 @@ type hcsNotification uint32 type notificationChannel chan error type notifcationWatcherContext struct { - channel notificationChannel - expectedNotification hcsNotification - handle hcsCallback + channels notificationChannels + handle hcsCallback +} + +type notificationChannels map[hcsNotification]notificationChannel + +func newChannels() notificationChannels { + channels := make(notificationChannels) + + channels[hcsNotificationSystemExited] = make(notificationChannel, 1) + channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1) + channels[hcsNotificationProcessExited] = make(notificationChannel, 1) + channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1) + return channels } func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr { - var ( - result error - completeWait = false - ) + var result error + if int32(notificationStatus) < 0 { + result = syscall.Errno(win32FromHresult(notificationStatus)) + } callbackMapLock.RLock() - context := callbackMap[callbackNumber] + channels := callbackMap[callbackNumber].channels callbackMapLock.RUnlock() - if notificationType == context.expectedNotification { - if int32(notificationStatus) < 0 { - result = syscall.Errno(win32FromHresult(notificationStatus)) - } else { - result = nil - } - completeWait = true - } else if notificationType == hcsNotificationSystemExited { - result = ErrUnexpectedContainerExit - completeWait = true - } else if notificationType == hcsNotificationServiceDisconnect { - result = ErrUnexpectedProcessAbort - completeWait = true - } - - if completeWait { - context.channel <- result - } + channels[notificationType] <- result return 0 } diff --git a/vendor/src/github.com/Microsoft/hcsshim/container.go b/vendor/src/github.com/Microsoft/hcsshim/container.go index 24b2327ddf..ab3ffbba25 100644 --- a/vendor/src/github.com/Microsoft/hcsshim/container.go +++ b/vendor/src/github.com/Microsoft/hcsshim/container.go @@ -26,8 +26,9 @@ type ContainerError struct { } type container struct { - handle hcsSystem - id string + handle hcsSystem + id string + callbackNumber uintptr } type containerProperties struct { @@ -47,7 +48,6 @@ type containerProperties struct { func CreateContainer(id string, c *ContainerConfig) (Container, error) { operation := "CreateContainer" title := "HCSShim::" + operation - logrus.Debugf(title+" id=%s", id) container := &container{ id: id, @@ -59,21 +59,28 @@ func CreateContainer(id string, c *ContainerConfig) (Container, error) { } configuration := string(configurationb) + logrus.Debugf(title+" id=%s config=%s", id, configuration) var ( - handle hcsSystem resultp *uint16 createError error ) if hcsCallbacksSupported { var identity syscall.Handle - createError = hcsCreateComputeSystem(id, configuration, identity, &handle, &resultp) - } else { - createError = hcsCreateComputeSystemTP5(id, configuration, &handle, &resultp) - } - container.handle = handle + createError = hcsCreateComputeSystem(id, configuration, identity, &container.handle, &resultp) - err = processAsyncHcsResult(container, createError, resultp, hcsNotificationSystemCreateCompleted, &defaultTimeout) + if createError == nil || createError == ErrVmcomputeOperationPending { + if err := container.registerCallback(); err != nil { + err := &ContainerError{Container: container, Operation: operation, Err: err} + logrus.Error(err) + return nil, err + } + } + } else { + createError = hcsCreateComputeSystemTP5(id, configuration, &container.handle, &resultp) + } + + err = processAsyncHcsResult(createError, resultp, container.callbackNumber, hcsNotificationSystemCreateCompleted, &defaultTimeout) if err != nil { err := &ContainerError{Container: container, Operation: operation, ExtraInfo: configuration, Err: err} logrus.Error(err) @@ -122,7 +129,7 @@ func (container *container) Start() error { var resultp *uint16 err := hcsStartComputeSystemTP5(container.handle, nil, &resultp) - err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemStartCompleted, &defaultTimeout) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemStartCompleted, &defaultTimeout) if err != nil { err := &ContainerError{Container: container, Operation: operation, Err: err} logrus.Error(err) @@ -186,7 +193,7 @@ func (container *container) Wait() error { logrus.Debugf(title+" id=%s", container.id) if hcsCallbacksSupported { - err := registerAndWaitForCallback(container, hcsNotificationSystemExited) + err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, nil) if err != nil { err := &ContainerError{Container: container, Operation: operation, Err: err} logrus.Error(err) @@ -217,7 +224,7 @@ func (container *container) WaitTimeout(timeout time.Duration) error { logrus.Debugf(title+" id=%s", container.id) if hcsCallbacksSupported { - err := registerAndWaitForCallbackTimeout(container, hcsNotificationSystemExited, timeout) + err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, &timeout) if err == ErrTimeout { return ErrTimeout } else if err != nil { @@ -304,7 +311,7 @@ func (container *container) Pause() error { var resultp *uint16 err := hcsPauseComputeSystemTP5(container.handle, nil, &resultp) - err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemPauseCompleted, &defaultTimeout) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemPauseCompleted, &defaultTimeout) if err != nil { err := &ContainerError{Container: container, Operation: operation, Err: err} logrus.Error(err) @@ -325,7 +332,7 @@ func (container *container) Resume() error { ) err := hcsResumeComputeSystemTP5(container.handle, nil, &resultp) - err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemResumeCompleted, &defaultTimeout) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemResumeCompleted, &defaultTimeout) if err != nil { err := &ContainerError{Container: container, Operation: operation, Err: err} logrus.Error(err) @@ -340,7 +347,6 @@ func (container *container) Resume() error { func (container *container) CreateProcess(c *ProcessConfig) (Process, error) { operation := "CreateProcess" title := "HCSShim::Container::" + operation - logrus.Debugf(title+" id=%s", container.id) var ( processInfo hcsProcessInformation processHandle hcsProcess @@ -359,6 +365,7 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) { } configuration := string(configurationb) + logrus.Debugf(title+" id=%s config=%s", container.id, configuration) err = hcsCreateProcess(container.handle, configuration, &processInfo, &processHandle, &resultp) err = processHcsResult(err, resultp) @@ -379,6 +386,14 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) { }, } + if hcsCallbacksSupported { + if err := process.registerCallback(); err != nil { + err = &ContainerError{Container: container, Operation: operation, Err: err} + logrus.Error(err) + return nil, err + } + } + logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID) runtime.SetFinalizer(process, closeProcess) return process, nil @@ -408,6 +423,12 @@ func (container *container) OpenProcess(pid int) (Process, error) { container: container, } + if err := process.registerCallback(); err != nil { + err = &ContainerError{Container: container, Operation: operation, Err: err} + logrus.Error(err) + return nil, err + } + logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID) runtime.SetFinalizer(process, closeProcess) return process, nil @@ -424,6 +445,14 @@ func (container *container) Close() error { return nil } + if hcsCallbacksSupported { + if err := container.unregisterCallback(); err != nil { + err = &ContainerError{Container: container, Operation: operation, Err: err} + logrus.Error(err) + return err + } + } + if err := hcsCloseComputeSystem(container.handle); err != nil { err = &ContainerError{Container: container, Operation: operation, Err: err} logrus.Error(err) @@ -441,7 +470,7 @@ func closeContainer(container *container) { container.Close() } -func (container *container) registerCallback(expectedNotification hcsNotification) (uintptr, error) { +func (container *container) registerCallback() error { callbackMapLock.Lock() defer callbackMapLock.Unlock() @@ -449,22 +478,24 @@ func (container *container) registerCallback(expectedNotification hcsNotificatio nextCallback++ context := ¬ifcationWatcherContext{ - expectedNotification: expectedNotification, - channel: make(chan error, 1), + channels: newChannels(), } callbackMap[callbackNumber] = context var callbackHandle hcsCallback err := hcsRegisterComputeSystemCallback(container.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) if err != nil { - return 0, err + return err } context.handle = callbackHandle + container.callbackNumber = callbackNumber - return callbackNumber, nil + return nil } -func (container *container) unregisterCallback(callbackNumber uintptr) error { +func (container *container) unregisterCallback() error { + callbackNumber := container.callbackNumber + callbackMapLock.Lock() defer callbackMapLock.Unlock() diff --git a/vendor/src/github.com/Microsoft/hcsshim/legacy.go b/vendor/src/github.com/Microsoft/hcsshim/legacy.go index b709f832cd..e19ac8a902 100644 --- a/vendor/src/github.com/Microsoft/hcsshim/legacy.go +++ b/vendor/src/github.com/Microsoft/hcsshim/legacy.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/binary" "errors" + "fmt" "io" "os" "path/filepath" @@ -77,7 +78,7 @@ func readTombstones(path string) (map[string]([]string), error) { ts := make(map[string]([]string)) for s.Scan() { - t := s.Text()[1:] // skip leading `\` + t := filepath.Join("Files", s.Text()[1:]) // skip leading `\` dir := filepath.Dir(t) ts[dir] = append(ts[dir], t) } @@ -107,6 +108,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error { if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { return nil } + r.result <- &fileEntry{path, info, nil} if !<-r.proceed { return errorIterationCanceled @@ -120,7 +122,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error { } if dts, ok := ts[relPath]; ok { for _, t := range dts { - r.result <- &fileEntry{t, nil, nil} + r.result <- &fileEntry{filepath.Join(r.root, t), nil, nil} if !<-r.proceed { return errorIterationCanceled } @@ -397,7 +399,10 @@ func (w *legacyLayerWriter) AddLink(name string, target string) error { } func (w *legacyLayerWriter) Remove(name string) error { - w.tombstones = append(w.tombstones, name) + if !strings.HasPrefix(name, `Files\`) { + return fmt.Errorf("invalid tombstone %s", name) + } + w.tombstones = append(w.tombstones, name[len(`Files\`):]) return nil } diff --git a/vendor/src/github.com/Microsoft/hcsshim/process.go b/vendor/src/github.com/Microsoft/hcsshim/process.go index a6941939f6..23ff39731c 100644 --- a/vendor/src/github.com/Microsoft/hcsshim/process.go +++ b/vendor/src/github.com/Microsoft/hcsshim/process.go @@ -22,11 +22,11 @@ type ProcessError struct { } type process struct { - handle hcsProcess - processID int - container *container - cachedPipes *cachedPipes - killCallbackNumber uintptr + handle hcsProcess + processID int + container *container + cachedPipes *cachedPipes + callbackNumber uintptr } type cachedPipes struct { @@ -101,7 +101,7 @@ func (process *process) Wait() error { logrus.Debugf(title+" processid=%d", process.processID) if hcsCallbacksSupported { - err := registerAndWaitForCallback(process, hcsNotificationProcessExited) + err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil) if err != nil { err := &ProcessError{Operation: operation, Process: process, Err: err} logrus.Error(err) @@ -128,7 +128,7 @@ func (process *process) WaitTimeout(timeout time.Duration) error { logrus.Debugf(title+" processid=%d", process.processID) if hcsCallbacksSupported { - err := registerAndWaitForCallbackTimeout(process, hcsNotificationProcessExited, timeout) + err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout) if err == ErrTimeout { return ErrTimeout } else if err != nil { @@ -344,6 +344,14 @@ func (process *process) Close() error { return nil } + if hcsCallbacksSupported { + if err := process.unregisterCallback(); err != nil { + err = &ProcessError{Operation: operation, Process: process, Err: err} + logrus.Error(err) + return err + } + } + if err := hcsCloseProcess(process.handle); err != nil { err = &ProcessError{Operation: operation, Process: process, Err: err} logrus.Error(err) @@ -361,7 +369,7 @@ func closeProcess(process *process) { process.Close() } -func (process *process) registerCallback(expectedNotification hcsNotification) (uintptr, error) { +func (process *process) registerCallback() error { callbackMapLock.Lock() defer callbackMapLock.Unlock() @@ -369,22 +377,24 @@ func (process *process) registerCallback(expectedNotification hcsNotification) ( nextCallback++ context := ¬ifcationWatcherContext{ - expectedNotification: expectedNotification, - channel: make(chan error, 1), + channels: newChannels(), } callbackMap[callbackNumber] = context var callbackHandle hcsCallback err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) if err != nil { - return 0, err + return err } context.handle = callbackHandle + process.callbackNumber = callbackNumber - return callbackNumber, nil + return nil } -func (process *process) unregisterCallback(callbackNumber uintptr) error { +func (process *process) unregisterCallback() error { + callbackNumber := process.callbackNumber + callbackMapLock.Lock() defer callbackMapLock.Unlock() handle := callbackMap[callbackNumber].handle diff --git a/vendor/src/github.com/Microsoft/hcsshim/waithelper.go b/vendor/src/github.com/Microsoft/hcsshim/waithelper.go index d36325a2df..d56798a5ee 100644 --- a/vendor/src/github.com/Microsoft/hcsshim/waithelper.go +++ b/vendor/src/github.com/Microsoft/hcsshim/waithelper.go @@ -1,6 +1,7 @@ package hcsshim import ( + "github.com/Sirupsen/logrus" "syscall" "time" ) @@ -10,11 +11,6 @@ type waitable interface { hcsWait(timeout uint32) (bool, error) } -type callbackable interface { - registerCallback(expectedNotification hcsNotification) (uintptr, error) - unregisterCallback(callbackNumber uintptr) error -} - func waitTimeoutHelper(object waitable, timeout time.Duration) (bool, error) { var ( millis uint32 @@ -52,62 +48,62 @@ func waitForSingleObject(handle syscall.Handle, timeout uint32) (bool, error) { } } -func processAsyncHcsResult(object callbackable, err error, resultp *uint16, expectedNotification hcsNotification, timeout *time.Duration) error { +func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { err = processHcsResult(err, resultp) if err == ErrVmcomputeOperationPending { - if timeout != nil { - err = registerAndWaitForCallbackTimeout(object, expectedNotification, *timeout) - } else { - err = registerAndWaitForCallback(object, expectedNotification) - } + return waitForNotification(callbackNumber, expectedNotification, timeout) } return err } -func registerAndWaitForCallbackTimeout(object callbackable, expectedNotification hcsNotification, timeout time.Duration) error { - callbackNumber, err := object.registerCallback(expectedNotification) - if err != nil { - return err - } - defer object.unregisterCallback(callbackNumber) - - return waitForNotificationTimeout(callbackNumber, timeout) -} - -func registerAndWaitForCallback(object callbackable, expectedNotification hcsNotification) error { - callbackNumber, err := object.registerCallback(expectedNotification) - if err != nil { - return err - } - defer object.unregisterCallback(callbackNumber) - - return waitForNotification(callbackNumber) -} - -func waitForNotificationTimeout(callbackNumber uintptr, timeout time.Duration) error { +func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { callbackMapLock.RLock() - channel := callbackMap[callbackNumber].channel + channels := callbackMap[callbackNumber].channels callbackMapLock.RUnlock() - timer := time.NewTimer(timeout) - defer timer.Stop() + expectedChannel := channels[expectedNotification] + if expectedChannel == nil { + logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification) + } + if timeout != nil { + timer := time.NewTimer(*timeout) + defer timer.Stop() + + select { + case err := <-expectedChannel: + return err + case err := <-channels[hcsNotificationSystemExited]: + // If the expected notification is hcsNotificationSystemExited which of the two selects + // chosen is random. Return the raw error if hcsNotificationSystemExited is expected + if channels[hcsNotificationSystemExited] == expectedChannel { + return err + } + return ErrUnexpectedContainerExit + case err := <-channels[hcsNotificationServiceDisconnect]: + // hcsNotificationServiceDisconnect should never be an expected notification + // it does not need the same handling as hcsNotificationSystemExited + logrus.Error(err) + return ErrUnexpectedProcessAbort + case <-timer.C: + return ErrTimeout + } + } select { - case err := <-channel: - return err - case <-timer.C: - return ErrTimeout - } -} - -func waitForNotification(callbackNumber uintptr) error { - callbackMapLock.RLock() - channel := callbackMap[callbackNumber].channel - callbackMapLock.RUnlock() - - select { - case err := <-channel: + case err := <-expectedChannel: return err + case err := <-channels[hcsNotificationSystemExited]: + // If the expected notification is hcsNotificationSystemExited which of the two selects + // chosen is random. Return the raw error if hcsNotificationSystemExited is expected + if channels[hcsNotificationSystemExited] == expectedChannel { + return err + } + return ErrUnexpectedContainerExit + case err := <-channels[hcsNotificationServiceDisconnect]: + // hcsNotificationServiceDisconnect should never be an expected notification + // it does not need the same handling as hcsNotificationSystemExited + logrus.Error(err) + return ErrUnexpectedProcessAbort } }