|
@@ -27,18 +27,24 @@ type discoveryReloader interface {
|
|
|
discovery.Watcher
|
|
|
Stop()
|
|
|
Reload(backend, address string, clusterOpts map[string]string) error
|
|
|
+ ReadyCh() <-chan struct{}
|
|
|
}
|
|
|
|
|
|
type daemonDiscoveryReloader struct {
|
|
|
backend discovery.Backend
|
|
|
ticker *time.Ticker
|
|
|
term chan bool
|
|
|
+ readyCh chan struct{}
|
|
|
}
|
|
|
|
|
|
func (d *daemonDiscoveryReloader) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
|
|
|
return d.backend.Watch(stopCh)
|
|
|
}
|
|
|
|
|
|
+func (d *daemonDiscoveryReloader) ReadyCh() <-chan struct{} {
|
|
|
+ return d.readyCh
|
|
|
+}
|
|
|
+
|
|
|
func discoveryOpts(clusterOpts map[string]string) (time.Duration, time.Duration, error) {
|
|
|
var (
|
|
|
heartbeat = defaultDiscoveryHeartbeat
|
|
@@ -87,38 +93,64 @@ func initDiscovery(backendAddress, advertiseAddress string, clusterOpts map[stri
|
|
|
backend: backend,
|
|
|
ticker: time.NewTicker(heartbeat),
|
|
|
term: make(chan bool),
|
|
|
+ readyCh: make(chan struct{}),
|
|
|
}
|
|
|
// We call Register() on the discovery backend in a loop for the whole lifetime of the daemon,
|
|
|
// but we never actually Watch() for nodes appearing and disappearing for the moment.
|
|
|
- reloader.advertise(advertiseAddress)
|
|
|
+ go reloader.advertiseHeartbeat(advertiseAddress)
|
|
|
return reloader, nil
|
|
|
}
|
|
|
|
|
|
-func (d *daemonDiscoveryReloader) advertise(address string) {
|
|
|
- d.registerAddr(address)
|
|
|
- go d.advertiseHeartbeat(address)
|
|
|
-}
|
|
|
-
|
|
|
-func (d *daemonDiscoveryReloader) registerAddr(addr string) {
|
|
|
- if err := d.backend.Register(addr); err != nil {
|
|
|
- log.Warnf("Registering as %q in discovery failed: %v", addr, err)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
// advertiseHeartbeat registers the current node against the discovery backend using the specified
|
|
|
// address. The function never returns, as registration against the backend comes with a TTL and
|
|
|
// requires regular heartbeats.
|
|
|
func (d *daemonDiscoveryReloader) advertiseHeartbeat(address string) {
|
|
|
+ var ready bool
|
|
|
+ if err := d.initHeartbeat(address); err == nil {
|
|
|
+ ready = true
|
|
|
+ close(d.readyCh)
|
|
|
+ }
|
|
|
+
|
|
|
for {
|
|
|
select {
|
|
|
case <-d.ticker.C:
|
|
|
- d.registerAddr(address)
|
|
|
+ if err := d.backend.Register(address); err != nil {
|
|
|
+ log.Warnf("Registering as %q in discovery failed: %v", address, err)
|
|
|
+ } else {
|
|
|
+ if !ready {
|
|
|
+ close(d.readyCh)
|
|
|
+ ready = true
|
|
|
+ }
|
|
|
+ }
|
|
|
case <-d.term:
|
|
|
return
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+// initHeartbeat is used to do the first heartbeat. It uses a tight loop until
|
|
|
+// either the timeout period is reached or the heartbeat is successful and returns.
|
|
|
+func (d *daemonDiscoveryReloader) initHeartbeat(address string) error {
|
|
|
+ // Setup a short ticker until the first heartbeat has succeeded
|
|
|
+ t := time.NewTicker(500 * time.Millisecond)
|
|
|
+ defer t.Stop()
|
|
|
+ // timeout makes sure that after a period of time we stop being so aggressive trying to reach the discovery service
|
|
|
+ timeout := time.After(60 * time.Second)
|
|
|
+
|
|
|
+ for {
|
|
|
+ select {
|
|
|
+ case <-timeout:
|
|
|
+ return errors.New("timeout waiting for initial discovery")
|
|
|
+ case <-d.term:
|
|
|
+ return errors.New("terminated")
|
|
|
+ case <-t.C:
|
|
|
+ if err := d.backend.Register(address); err == nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
// Reload makes the watcher to stop advertising and reconfigures it to advertise in a new address.
|
|
|
func (d *daemonDiscoveryReloader) Reload(backendAddress, advertiseAddress string, clusterOpts map[string]string) error {
|
|
|
d.Stop()
|
|
@@ -130,8 +162,9 @@ func (d *daemonDiscoveryReloader) Reload(backendAddress, advertiseAddress string
|
|
|
|
|
|
d.backend = backend
|
|
|
d.ticker = time.NewTicker(heartbeat)
|
|
|
+ d.readyCh = make(chan struct{})
|
|
|
|
|
|
- d.advertise(advertiseAddress)
|
|
|
+ go d.advertiseHeartbeat(advertiseAddress)
|
|
|
return nil
|
|
|
}
|
|
|
|