moby/integration-cli/docker_cli_service_health_test.go

// +build !windows

package main

import (
	"strconv"
	"strings"

	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/daemon/cluster/executor/container"
	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

// start a service, and then make its task unhealthy during running
// finally, unhealthy task should be detected and killed
func (s *DockerSwarmSuite) TestServiceHealthRun(c *check.C) {
	testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows

	d := s.AddDaemon(c, true, true)

	// build image with health-check
	// note: use `daemon.buildImageWithOut` to build, do not use `buildImage` to build
	imageName := "testhealth"
	_, _, err := d.buildImageWithOut(imageName,
		`FROM busybox
		RUN touch /status
		HEALTHCHECK --interval=1s --timeout=1s --retries=1\
		  CMD cat /status`,
		true)
	c.Check(err, check.IsNil)

	serviceName := "healthServiceRun"
	out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top")
	c.Assert(err, checker.IsNil, check.Commentf(out))
	id := strings.TrimSpace(out)

	var tasks []swarm.Task
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		tasks = d.getServiceTasks(c, id)
		return tasks, nil
	}, checker.HasLen, 1)

	task := tasks[0]

	// wait for task to start
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateRunning)
	containerID := task.Status.ContainerStatus.ContainerID

	// wait for container to be healthy
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		out, _ := d.Cmd("inspect", "--format={{.State.Health.Status}}", containerID)
		return strings.TrimSpace(out), nil
	}, checker.Equals, "healthy")

	// make it fail
	d.Cmd("exec", containerID, "rm", "/status")
	// wait for container to be unhealthy
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		out, _ := d.Cmd("inspect", "--format={{.State.Health.Status}}", containerID)
		return strings.TrimSpace(out), nil
	}, checker.Equals, "unhealthy")

	// Task should be terminated
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateFailed)

	if !strings.Contains(task.Status.Err, container.ErrContainerUnhealthy.Error()) {
		c.Fatal("unhealthy task exits because of other error")
	}
}

// start a service whose task is unhealthy at beginning
// its tasks should be blocked in starting stage, until health check is passed
func (s *DockerSwarmSuite) TestServiceHealthStart(c *check.C) {
	testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows

	d := s.AddDaemon(c, true, true)

	// service started from this image won't pass health check
	imageName := "testhealth"
	_, _, err := d.buildImageWithOut(imageName,
		`FROM busybox
		HEALTHCHECK --interval=1s --timeout=1s --retries=1024\
		  CMD cat /status`,
		true)
	c.Check(err, check.IsNil)

	serviceName := "healthServiceStart"
	out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top")
	c.Assert(err, checker.IsNil, check.Commentf(out))
	id := strings.TrimSpace(out)

	var tasks []swarm.Task
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		tasks = d.getServiceTasks(c, id)
		return tasks, nil
	}, checker.HasLen, 1)

	task := tasks[0]

	// wait for task to start
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateStarting)

	containerID := task.Status.ContainerStatus.ContainerID

	// wait for health check to work
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		out, _ := d.Cmd("inspect", "--format={{.State.Health.FailingStreak}}", containerID)
		failingStreak, _ := strconv.Atoi(strings.TrimSpace(out))
		return failingStreak, nil
	}, checker.GreaterThan, 0)

	// task should be blocked at starting status
	task = d.getTask(c, task.ID)
	c.Assert(task.Status.State, check.Equals, swarm.TaskStateStarting)

	// make it healthy
	d.Cmd("exec", containerID, "touch", "/status")

	// Task should be at running status
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateRunning)
}

// start a service whose task is unhealthy at beginning
// its tasks should be blocked in starting stage, until health check is passed
func (s *DockerSwarmSuite) TestServiceHealthUpdate(c *check.C) {
	testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows

	d := s.AddDaemon(c, true, true)

	// service started from this image won't pass health check
	imageName := "testhealth"
	_, _, err := d.buildImageWithOut(imageName,
		`FROM busybox
		HEALTHCHECK --interval=1s --timeout=1s --retries=1024\
		  CMD cat /status`,
		true)
	c.Check(err, check.IsNil)

	serviceName := "healthServiceStart"
	out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top")
	c.Assert(err, checker.IsNil, check.Commentf(out))
	id := strings.TrimSpace(out)

	var tasks []swarm.Task
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		tasks = d.getServiceTasks(c, id)
		return tasks, nil
	}, checker.HasLen, 1)

	task := tasks[0]

	// wait for task to start
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateStarting)

	containerID := task.Status.ContainerStatus.ContainerID

	// wait for health check to work
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		out, _ := d.Cmd("inspect", "--format={{.State.Health.FailingStreak}}", containerID)
		failingStreak, _ := strconv.Atoi(strings.TrimSpace(out))
		return failingStreak, nil
	}, checker.GreaterThan, 0)

	// task should be blocked at starting status
	task = d.getTask(c, task.ID)
	c.Assert(task.Status.State, check.Equals, swarm.TaskStateStarting)

	// make it healthy
	d.Cmd("exec", containerID, "touch", "/status")
	// Task should be at running status
	waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) {
		task = d.getTask(c, task.ID)
		return task.Status.State, nil
	}, checker.Equals, swarm.TaskStateRunning)
}