Update BuildKit registry config on daemon reload

Historically, daemon.RegistryHosts() has returned a docker.RegistryHosts
callback function which closes over a point-in-time snapshot of the
daemon configuration. When constructing the BuildKit builder at daemon
startup, the return value of daemon.RegistryHosts() has been used.
Therefore the BuildKit builder would use the registry configuration as
it was at daemon startup for the life of the process, even if the
registry configuration is changed and the configuration reloaded.
Provide BuildKit with a RegistryHosts callback which reflects the
live daemon configuration after reloads so that registry operations
performed by BuildKit always use the same configuration as the rest of
the daemon.

Signed-off-by: Cory Snider <csnider@mirantis.com>

cmd/dockerd/daemon.go

@@ -369,7 +369,7 @@ func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daem
 		ImageTagger:         d.ImageService(),
 		NetworkController:   d.NetworkController(),
 		DefaultCgroupParent: cgroupParent,
-		RegistryHosts:       d.RegistryHosts(),
+		RegistryHosts:       d.RegistryHosts,
 		BuilderConfig:       config.Builder,
 		Rootless:            d.Rootless(),
 		IdentityMapping:     d.IdentityMapping(),

daemon/containerd/resolver.go

@@ -16,9 +16,8 @@ import (
 
 func (i *ImageService) newResolverFromAuthConfig(ctx context.Context, authConfig *registrytypes.AuthConfig) (remotes.Resolver, docker.StatusTracker) {
 	tracker := docker.NewInMemoryTracker()
-	hostsFn := i.registryHosts.RegistryHosts()
 
-	hosts := hostsWrapper(hostsFn, authConfig, i.registryService)
+	hosts := hostsWrapper(i.registryHosts, authConfig, i.registryService)
 	headers := http.Header{}
 	headers.Set("User-Agent", dockerversion.DockerUserAgent(ctx))
 

daemon/containerd/service.go

@@ -31,16 +31,12 @@ type ImageService struct {
 	client          *containerd.Client
 	containers      container.Store
 	snapshotter     string
-	registryHosts   RegistryHostsProvider
+	registryHosts   docker.RegistryHosts
 	registryService RegistryConfigProvider
 	eventsService   *daemonevents.Events
 	pruneRunning    atomic.Bool
 }
 
-type RegistryHostsProvider interface {
-	RegistryHosts() docker.RegistryHosts
-}
-
 type RegistryConfigProvider interface {
 	IsInsecureRegistry(host string) bool
 	ResolveRepository(name reference.Named) (*registry.RepositoryInfo, error)
@@ -50,7 +46,7 @@ type ImageServiceConfig struct {
 	Client        *containerd.Client
 	Containers    container.Store
 	Snapshotter   string
-	HostsProvider RegistryHostsProvider
+	RegistryHosts docker.RegistryHosts
 	Registry      RegistryConfigProvider
 	EventsService *daemonevents.Events
 }
@@ -61,7 +57,7 @@ func NewService(config ImageServiceConfig) *ImageService {
 		client:          config.Client,
 		containers:      config.Containers,
 		snapshotter:     config.Snapshotter,
-		registryHosts:   config.HostsProvider,
+		registryHosts:   config.RegistryHosts,
 		registryService: config.Registry,
 		eventsService:   config.EventsService,
 	}

daemon/daemon.go

@@ -169,15 +169,20 @@ func (daemon *Daemon) UsesSnapshotter() bool {
 	return daemon.usesSnapshotter
 }
 
-// RegistryHosts returns registry configuration in containerd resolvers format
-func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
+// RegistryHosts returns the registry hosts configuration for the host component
+// of a distribution image reference.
+func (daemon *Daemon) RegistryHosts(host string) ([]docker.RegistryHost, error) {
+	daemon.configStore.Lock()
+	serviceOpts := daemon.configStore.ServiceOptions
+	daemon.configStore.Unlock()
+
 	var (
 		registryKey = "docker.io"
-		mirrors     = make([]string, len(daemon.configStore.Mirrors))
+		mirrors     = make([]string, len(serviceOpts.Mirrors))
 		m           = map[string]resolverconfig.RegistryConfig{}
 	)
 	// must trim "https://" or "http://" prefix
-	for i, v := range daemon.configStore.Mirrors {
+	for i, v := range serviceOpts.Mirrors {
 		if uri, err := url.Parse(v); err == nil {
 			v = uri.Host
 		}
@@ -186,7 +191,7 @@ func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
 	// set mirrors for default registry
 	m[registryKey] = resolverconfig.RegistryConfig{Mirrors: mirrors}
 
-	for _, v := range daemon.configStore.InsecureRegistries {
+	for _, v := range serviceOpts.InsecureRegistries {
 		u, err := url.Parse(v)
 		if err != nil && !strings.HasPrefix(v, "http://") && !strings.HasPrefix(v, "https://") {
 			originalErr := err
@@ -224,7 +229,7 @@ func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
 		}
 	}
 
-	return resolver.NewRegistryConfig(m)
+	return resolver.NewRegistryConfig(m)(host)
 }
 
 // layerAccessor may be implemented by ImageService
@@ -1032,7 +1037,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 			Client:        d.containerdCli,
 			Containers:    d.containers,
 			Snapshotter:   driverName,
-			HostsProvider: d,
+			RegistryHosts: d.RegistryHosts,
 			Registry:      d.registryService,
 			EventsService: d.EventsService,
 		})