Migrate TestAPISwarmServicesPlugin to integration

Also starts to create more "poll/check" function to `internal/test/daemon`.

Signed-off-by: Vincent Demeester <vincent@sbr.pm>

integration-cli/docker_api_swarm_service_test.go

@@ -4,19 +4,15 @@ package main
 
 import (
 	"fmt"
-	"path"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/swarm"
-	"github.com/docker/docker/api/types/swarm/runtime"
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/daemon"
 	testdaemon "github.com/docker/docker/internal/test/daemon"
-	"github.com/docker/docker/internal/test/fixtures/plugin"
-	"github.com/docker/docker/internal/test/registry"
 	"github.com/go-check/check"
 	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
@@ -611,78 +607,3 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) {
 		}
 	}
 }
-
-// Test plugins deployed via swarm services
-func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) {
-	testRequires(c, ExperimentalDaemon, DaemonIsLinux, IsAmd64)
-
-	reg := registry.NewV2(c)
-	defer reg.Close()
-
-	repo := path.Join(privateRegistryURL, "swarm", "test:v1")
-	repo2 := path.Join(privateRegistryURL, "swarm", "test:v2")
-	name := "test"
-
-	err := plugin.CreateInRegistry(context.Background(), repo, nil)
-	c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin"))
-	err = plugin.CreateInRegistry(context.Background(), repo2, nil)
-	c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin"))
-
-	d1 := s.AddDaemon(c, true, true)
-	d2 := s.AddDaemon(c, true, true)
-	d3 := s.AddDaemon(c, true, false)
-
-	makePlugin := func(repo, name string, constraints []string) func(*swarm.Service) {
-		return func(s *swarm.Service) {
-			s.Spec.TaskTemplate.Runtime = "plugin"
-			s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{
-				Name:   name,
-				Remote: repo,
-			}
-			if constraints != nil {
-				s.Spec.TaskTemplate.Placement = &swarm.Placement{
-					Constraints: constraints,
-				}
-			}
-		}
-	}
-
-	id := d1.CreateService(c, makePlugin(repo, name, nil))
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True)
-
-	service := d1.GetService(c, id)
-	d1.UpdateService(c, service, makePlugin(repo2, name, nil))
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginImage(name), checker.Equals, repo2)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginImage(name), checker.Equals, repo2)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginImage(name), checker.Equals, repo2)
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True)
-
-	d1.RemoveService(c, id)
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False)
-
-	// constrain to managers only
-	id = d1.CreateService(c, makePlugin(repo, name, []string{"node.role==manager"}))
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) // Not a manager, not running it
-	d1.RemoveService(c, id)
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False)
-
-	// with no name
-	id = d1.CreateService(c, makePlugin(repo, "", nil))
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.True)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.True)
-	d1.RemoveService(c, id)
-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.False)
-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.False)
-}

integration/internal/swarm/service.go

@@ -19,10 +19,10 @@ import (
 // ServicePoll tweaks the pollSettings for `service`
 func ServicePoll(config *poll.Settings) {
 	// Override the default pollSettings for `service` resource here ...
-
+	config.Timeout = 30 * time.Second
+	config.Delay = 100 * time.Millisecond
 	if runtime.GOARCH == "arm64" || runtime.GOARCH == "arm" {
 		config.Timeout = 1 * time.Minute
-		config.Delay = 100 * time.Millisecond
 	}
 }
 

integration/service/plugin_test.go

@@ -0,0 +1,120 @@
+package service
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	swarmtypes "github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/swarm/runtime"
+	"github.com/docker/docker/integration/internal/swarm"
+	"github.com/docker/docker/internal/test/daemon"
+	"github.com/docker/docker/internal/test/fixtures/plugin"
+	"github.com/docker/docker/internal/test/registry"
+	"github.com/gotestyourself/gotestyourself/assert"
+	"github.com/gotestyourself/gotestyourself/poll"
+	"github.com/gotestyourself/gotestyourself/skip"
+)
+
+func TestServicePlugin(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, os.Getenv("DOCKER_ENGINE_GOARCH") != "amd64")
+	defer setupTest(t)()
+
+	reg := registry.NewV2(t)
+	defer reg.Close()
+
+	repo := path.Join(registry.DefaultURL, "swarm", "test:v1")
+	repo2 := path.Join(registry.DefaultURL, "swarm", "test:v2")
+	name := "test"
+
+	d := daemon.New(t)
+	d.StartWithBusybox(t)
+	apiclient := d.NewClientT(t)
+	err := plugin.Create(context.Background(), apiclient, repo)
+	assert.NilError(t, err)
+	r, err := apiclient.PluginPush(context.Background(), repo, "")
+	assert.NilError(t, err)
+	_, err = io.Copy(ioutil.Discard, r)
+	assert.NilError(t, err)
+	err = apiclient.PluginRemove(context.Background(), repo, types.PluginRemoveOptions{})
+	assert.NilError(t, err)
+	err = plugin.Create(context.Background(), apiclient, repo2)
+	assert.NilError(t, err)
+	r, err = apiclient.PluginPush(context.Background(), repo2, "")
+	assert.NilError(t, err)
+	_, err = io.Copy(ioutil.Discard, r)
+	assert.NilError(t, err)
+	err = apiclient.PluginRemove(context.Background(), repo2, types.PluginRemoveOptions{})
+	assert.NilError(t, err)
+	d.Stop(t)
+
+	d1 := swarm.NewSwarm(t, testEnv, daemon.WithExperimental)
+	defer d1.Stop(t)
+	d2 := daemon.New(t, daemon.WithExperimental, daemon.WithSwarmPort(daemon.DefaultSwarmPort+1))
+	d2.StartAndSwarmJoin(t, d1, true)
+	defer d2.Stop(t)
+	d3 := daemon.New(t, daemon.WithExperimental, daemon.WithSwarmPort(daemon.DefaultSwarmPort+2))
+	d3.StartAndSwarmJoin(t, d1, false)
+	defer d3.Stop(t)
+
+	id := d1.CreateService(t, makePlugin(repo, name, nil))
+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsRunning(name), swarm.ServicePoll)
+
+	service := d1.GetService(t, id)
+	d1.UpdateService(t, service, makePlugin(repo2, name, nil))
+	poll.WaitOn(t, d1.PluginReferenceIs(name, repo2), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginReferenceIs(name, repo2), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginReferenceIs(name, repo2), swarm.ServicePoll)
+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsRunning(name), swarm.ServicePoll)
+
+	d1.RemoveService(t, id)
+	poll.WaitOn(t, d1.PluginIsNotPresent(name), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsNotPresent(name), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)
+
+	// constrain to managers only
+	id = d1.CreateService(t, makePlugin(repo, name, []string{"node.role==manager"}))
+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)
+
+	d1.RemoveService(t, id)
+	poll.WaitOn(t, d1.PluginIsNotPresent(name), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsNotPresent(name), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)
+
+	// with no name
+	id = d1.CreateService(t, makePlugin(repo, "", nil))
+	poll.WaitOn(t, d1.PluginIsRunning(repo), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsRunning(repo), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsRunning(repo), swarm.ServicePoll)
+
+	d1.RemoveService(t, id)
+	poll.WaitOn(t, d1.PluginIsNotPresent(repo), swarm.ServicePoll)
+	poll.WaitOn(t, d2.PluginIsNotPresent(repo), swarm.ServicePoll)
+	poll.WaitOn(t, d3.PluginIsNotPresent(repo), swarm.ServicePoll)
+}
+
+func makePlugin(repo, name string, constraints []string) func(*swarmtypes.Service) {
+	return func(s *swarmtypes.Service) {
+		s.Spec.TaskTemplate.Runtime = "plugin"
+		s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{
+			Name:   name,
+			Remote: repo,
+		}
+		if constraints != nil {
+			s.Spec.TaskTemplate.Placement = &swarmtypes.Placement{
+				Constraints: constraints,
+			}
+		}
+	}
+}

internal/test/daemon/daemon.go

@@ -113,7 +113,7 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon {
 		execRoot:        filepath.Join(os.TempDir(), "docker-execroot", id),
 		dockerdBinary:   defaultDockerdBinary,
 		swarmListenAddr: defaultSwarmListenAddr,
-		SwarmPort:       defaultSwarmPort,
+		SwarmPort:       DefaultSwarmPort,
 		log:             t,
 	}
 

internal/test/daemon/plugin.go

@@ -0,0 +1,77 @@
+package daemon
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/gotestyourself/gotestyourself/poll"
+)
+
+// PluginIsRunning provides a poller to check if the specified plugin is running
+func (d *Daemon) PluginIsRunning(name string) func(poll.LogT) poll.Result {
+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {
+		if plugin.Enabled {
+			return poll.Success()
+		}
+		return poll.Continue("plugin %q is not enabled", name)
+	}))
+}
+
+// PluginIsNotRunning provides a poller to check if the specified plugin is not running
+func (d *Daemon) PluginIsNotRunning(name string) func(poll.LogT) poll.Result {
+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {
+		if !plugin.Enabled {
+			return poll.Success()
+		}
+		return poll.Continue("plugin %q is enabled", name)
+	}))
+}
+
+// PluginIsNotPresent provides a poller to check if the specified plugin is not present
+func (d *Daemon) PluginIsNotPresent(name string) func(poll.LogT) poll.Result {
+	return withClient(d, func(c client.APIClient, t poll.LogT) poll.Result {
+		_, _, err := c.PluginInspectWithRaw(context.Background(), name)
+		if client.IsErrNotFound(err) {
+			return poll.Success()
+		}
+		if err != nil {
+			return poll.Error(err)
+		}
+		return poll.Continue("plugin %q exists")
+	})
+}
+
+// PluginReferenceIs provides a poller to check if the specified plugin has the specified reference
+func (d *Daemon) PluginReferenceIs(name, expectedRef string) func(poll.LogT) poll.Result {
+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {
+		if plugin.PluginReference == expectedRef {
+			return poll.Success()
+		}
+		return poll.Continue("plugin %q reference is not %q", name, expectedRef)
+	}))
+}
+
+func withPluginInspect(name string, f func(*types.Plugin, poll.LogT) poll.Result) func(client.APIClient, poll.LogT) poll.Result {
+	return func(c client.APIClient, t poll.LogT) poll.Result {
+		plugin, _, err := c.PluginInspectWithRaw(context.Background(), name)
+		if client.IsErrNotFound(err) {
+			return poll.Continue("plugin %q not found", name)
+		}
+		if err != nil {
+			return poll.Error(err)
+		}
+		return f(plugin, t)
+	}
+
+}
+
+func withClient(d *Daemon, f func(client.APIClient, poll.LogT) poll.Result) func(poll.LogT) poll.Result {
+	return func(t poll.LogT) poll.Result {
+		c, err := d.NewClient()
+		if err != nil {
+			poll.Error(err)
+		}
+		return f(c, t)
+	}
+}

internal/test/daemon/swarm.go

@@ -10,7 +10,8 @@ import (
 )
 
 const (
-	defaultSwarmPort       = 2477
+	// DefaultSwarmPort is the default port use for swarm in the tests
+	DefaultSwarmPort       = 2477
 	defaultSwarmListenAddr = "0.0.0.0"
 )