Make test suites dump daemon stack on test timeout

Use `OnTimeout` callback on test timeouts to trigger a stack dump for
running daemons. This will help analyze stuck tests.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

integration-cli/check_test.go

@@ -34,6 +34,12 @@ func init() {
 type DockerSuite struct {
 }
 
+func (s *DockerSuite) OnTimeout(c *check.C) {
+	if daemonPid > 0 && isLocalDaemon {
+		signalDaemonDump(daemonPid)
+	}
+}
+
 func (s *DockerSuite) TearDownTest(c *check.C) {
 	unpauseAllContainers()
 	deleteAllContainers()
@@ -54,6 +60,10 @@ type DockerRegistrySuite struct {
 	d   *Daemon
 }
 
+func (s *DockerRegistrySuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
 func (s *DockerRegistrySuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux, RegistryHosting)
 	s.reg = setupRegistry(c, false, "", "")
@@ -82,6 +92,10 @@ type DockerSchema1RegistrySuite struct {
 	d   *Daemon
 }
 
+func (s *DockerSchema1RegistrySuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
 func (s *DockerSchema1RegistrySuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux, RegistryHosting, NotArm64)
 	s.reg = setupRegistry(c, true, "", "")
@@ -110,6 +124,10 @@ type DockerRegistryAuthHtpasswdSuite struct {
 	d   *Daemon
 }
 
+func (s *DockerRegistryAuthHtpasswdSuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
 func (s *DockerRegistryAuthHtpasswdSuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux, RegistryHosting)
 	s.reg = setupRegistry(c, false, "htpasswd", "")
@@ -140,6 +158,10 @@ type DockerRegistryAuthTokenSuite struct {
 	d   *Daemon
 }
 
+func (s *DockerRegistryAuthTokenSuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
 func (s *DockerRegistryAuthTokenSuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux, RegistryHosting)
 	s.d = NewDaemon(c)
@@ -175,6 +197,10 @@ type DockerDaemonSuite struct {
 	d  *Daemon
 }
 
+func (s *DockerDaemonSuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
 func (s *DockerDaemonSuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	s.d = NewDaemon(c)
@@ -218,6 +244,14 @@ type DockerSwarmSuite struct {
 	portIndex   int
 }
 
+func (s *DockerSwarmSuite) OnTimeout(c *check.C) {
+	s.daemonsLock.Lock()
+	defer s.daemonsLock.Unlock()
+	for _, d := range s.daemons {
+		d.DumpStackAndQuit()
+	}
+}
+
 func (s *DockerSwarmSuite) SetUpTest(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 }

integration-cli/daemon.go

@@ -273,6 +273,16 @@ func (d *Daemon) Kill() error {
 	return nil
 }
 
+// DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its
+// stack to its log file and exit
+// This is used primarily for gathering debug information on test timeout
+func (d *Daemon) DumpStackAndQuit() {
+	if d.cmd == nil || d.cmd.Process == nil {
+		return
+	}
+	signalDaemonDump(d.cmd.Process.Pid)
+}
+
 // Stop will send a SIGINT every second and wait for the daemon to stop.
 // If it timeouts, a SIGKILL is sent.
 // Stop will not delete the daemon directory. If a purged daemon is needed,

integration-cli/daemon_unix.go

@@ -0,0 +1,9 @@
+// +build !windows
+
+package main
+
+import "syscall"
+
+func signalDaemonDump(pid int) {
+	syscall.Kill(pid, syscall.SIGQUIT)
+}

integration-cli/daemon_windows.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"strconv"
+	"syscall"
+	"unsafe"
+)
+
+func openEvent(desiredAccess uint32, inheritHandle bool, name string, proc *syscall.LazyProc) (handle syscall.Handle, err error) {
+	namep, _ := syscall.UTF16PtrFromString(name)
+	var _p2 uint32
+	if inheritHandle {
+		_p2 = 1
+	}
+	r0, _, e1 := proc.Call(uintptr(desiredAccess), uintptr(_p2), uintptr(unsafe.Pointer(namep)))
+	handle = syscall.Handle(r0)
+	if handle == syscall.InvalidHandle {
+		err = e1
+	}
+	return
+}
+
+func pulseEvent(handle syscall.Handle, proc *syscall.LazyProc) (err error) {
+	r0, _, _ := proc.Call(uintptr(handle))
+	if r0 != 0 {
+		err = syscall.Errno(r0)
+	}
+	return
+}
+
+func signalDaemonDump(pid int) {
+	modkernel32 := syscall.NewLazyDLL("kernel32.dll")
+	procOpenEvent := modkernel32.NewProc("OpenEventW")
+	procPulseEvent := modkernel32.NewProc("PulseEvent")
+
+	ev := "Global\\docker-daemon-" + strconv.Itoa(pid)
+	h2, _ := openEvent(0x0002, false, ev, procOpenEvent)
+	if h2 == 0 {
+		return
+	}
+	pulseEvent(h2, procPulseEvent)
+}

integration-cli/docker_test_vars.go

@@ -3,8 +3,11 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strconv"
 
 	"github.com/docker/docker/pkg/reexec"
 )
@@ -65,6 +68,9 @@ var (
 	// WindowsBaseImage is the name of the base image for Windows testing
 	// Environment variable WINDOWS_BASE_IMAGE can override this
 	WindowsBaseImage = "windowsservercore"
+
+	// daemonPid is the pid of the main test daemon
+	daemonPid int
 )
 
 const (
@@ -134,4 +140,12 @@ func init() {
 		WindowsBaseImage = os.Getenv("WINDOWS_BASE_IMAGE")
 		fmt.Println("INFO: Windows Base image is ", WindowsBaseImage)
 	}
+
+	dest := os.Getenv("DEST")
+	b, err = ioutil.ReadFile(filepath.Join(dest, "docker.pid"))
+	if err == nil {
+		if p, err := strconv.ParseInt(string(b), 10, 32); err == nil {
+			daemonPid = int(p)
+		}
+	}
 }