diff --git a/libnetwork/controller.go b/libnetwork/controller.go
index 1c5026f461..0994ec8e8e 100644
--- a/libnetwork/controller.go
+++ b/libnetwork/controller.go
@@ -218,7 +218,9 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 
 func (c *controller) SetClusterProvider(provider cluster.Provider) {
 	c.cfg.Daemon.ClusterProvider = provider
-	go c.clusterAgentInit()
+	if provider != nil {
+		go c.clusterAgentInit()
+	}
 }
 
 func isValidClusteringIP(addr string) bool {
@@ -261,6 +263,7 @@ func (c *controller) clusterAgentInit() {
 			} else {
 				c.agentInitDone = make(chan struct{})
 				c.agentClose()
+				return
 			}
 		}
 	}