diff --git a/api/server/server.go b/api/server/server.go
index e8714eb060..3ce2659287 100644
--- a/api/server/server.go
+++ b/api/server/server.go
@@ -2,7 +2,6 @@ package server // import "github.com/docker/docker/api/server"
 
 import (
 	"context"
-	"crypto/tls"
 	"net"
 	"net/http"
 	"strings"
@@ -22,32 +21,13 @@ import (
 // when a request is about to be served.
 const versionMatcher = "/v{version:[0-9.]+}"
 
-// Config provides the configuration for the API server
-type Config struct {
-	CorsHeaders string
-	Version     string
-	SocketGroup string
-	TLSConfig   *tls.Config
-	// Hosts is a list of addresses for the API to listen on.
-	Hosts []string
-}
-
 // Server contains instance details for the server
 type Server struct {
-	cfg         *Config
 	servers     []*HTTPServer
 	routers     []router.Router
 	middlewares []middleware.Middleware
 }
 
-// New returns a new instance of the server based on the specified configuration.
-// It allocates resources which will be needed for ServeAPI(ports, unix-sockets).
-func New(cfg *Config) *Server {
-	return &Server{
-		cfg: cfg,
-	}
-}
-
 // UseMiddleware appends a new middleware to the request chain.
 // This needs to be called before the API routes are configured.
 func (s *Server) UseMiddleware(m middleware.Middleware) {
diff --git a/api/server/server_test.go b/api/server/server_test.go
index a3e8124a88..8b71e9fce6 100644
--- a/api/server/server_test.go
+++ b/api/server/server_test.go
@@ -13,12 +13,7 @@ import (
 )
 
 func TestMiddlewares(t *testing.T) {
-	cfg := &Config{
-		Version: "0.1omega2",
-	}
-	srv := &Server{
-		cfg: cfg,
-	}
+	srv := &Server{}
 
 	srv.UseMiddleware(middleware.NewVersionMiddleware("0.1omega2", api.DefaultVersion, api.MinVersion))
 
diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go
index 8389378174..90348f1217 100644
--- a/cmd/dockerd/daemon.go
+++ b/cmd/dockerd/daemon.go
@@ -65,7 +65,7 @@ type DaemonCli struct {
 	configFile *string
 	flags      *pflag.FlagSet
 
-	api             *apiserver.Server
+	api             apiserver.Server
 	d               *daemon.Daemon
 	authzMiddleware *authorization.Middleware // authzMiddleware enables to dynamically reload the authorization plugins
 }
@@ -80,7 +80,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return err
 	}
 
-	serverConfig, err := newAPIServerConfig(cli.Config)
+	tlsConfig, err := newAPIServerTLSConfig(cli.Config)
 	if err != nil {
 		return err
 	}
@@ -161,9 +161,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		}
 	}
 
-	cli.api = apiserver.New(serverConfig)
-
-	hosts, err := loadListeners(cli, serverConfig)
+	hosts, err := loadListeners(cli, tlsConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to load listeners")
 	}
@@ -192,7 +190,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 
 	pluginStore := plugin.NewStore()
 
-	if err := cli.initMiddlewares(cli.api, serverConfig, pluginStore); err != nil {
+	if err := cli.initMiddlewares(&cli.api, pluginStore); err != nil {
 		logrus.Fatalf("Error creating middlewares: %v", err)
 	}
 
@@ -230,7 +228,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 	if err != nil {
 		return err
 	}
-	routerOptions.api = cli.api
+	routerOptions.api = &cli.api
 	routerOptions.cluster = c
 
 	initRouter(routerOptions)
@@ -546,8 +544,8 @@ func initRouter(opts routerOptions) {
 }
 
 // TODO: remove this from cli and return the authzMiddleware
-func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config, pluginStore plugingetter.PluginGetter) error {
-	v := cfg.Version
+func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, pluginStore plugingetter.PluginGetter) error {
+	v := dockerversion.Version
 
 	exp := middleware.NewExperimentalMiddleware(cli.Config.Experimental)
 	s.UseMiddleware(exp)
@@ -555,8 +553,8 @@ func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config
 	vm := middleware.NewVersionMiddleware(v, api.DefaultVersion, api.MinVersion)
 	s.UseMiddleware(vm)
 
-	if cfg.CorsHeaders != "" {
-		c := middleware.NewCORSMiddleware(cfg.CorsHeaders)
+	if cli.Config.CorsHeaders != "" {
+		c := middleware.NewCORSMiddleware(cli.Config.CorsHeaders)
 		s.UseMiddleware(c)
 	}
 
@@ -597,7 +595,7 @@ func (cli *DaemonCli) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error)
 	return opts, nil
 }
 
-func newAPIServerConfig(config *config.Config) (*apiserver.Config, error) {
+func newAPIServerTLSConfig(config *config.Config) (*tls.Config, error) {
 	var tlsConfig *tls.Config
 	if config.TLS != nil && *config.TLS {
 		var (
@@ -620,13 +618,7 @@ func newAPIServerConfig(config *config.Config) (*apiserver.Config, error) {
 		}
 	}
 
-	return &apiserver.Config{
-		SocketGroup: config.SocketGroup,
-		Version:     dockerversion.Version,
-		CorsHeaders: config.CorsHeaders,
-		TLSConfig:   tlsConfig,
-		Hosts:       config.Hosts,
-	}, nil
+	return tlsConfig, nil
 }
 
 // checkTLSAuthOK checks basically for an explicitly disabled TLS/TLSVerify
@@ -654,21 +646,21 @@ func checkTLSAuthOK(c *config.Config) bool {
 	return true
 }
 
-func loadListeners(cli *DaemonCli, serverConfig *apiserver.Config) ([]string, error) {
-	if len(serverConfig.Hosts) == 0 {
+func loadListeners(cli *DaemonCli, tlsConfig *tls.Config) ([]string, error) {
+	if len(cli.Config.Hosts) == 0 {
 		return nil, errors.New("no hosts configured")
 	}
 	var hosts []string
 
-	for i := 0; i < len(serverConfig.Hosts); i++ {
-		protoAddr := serverConfig.Hosts[i]
+	for i := 0; i < len(cli.Config.Hosts); i++ {
+		protoAddr := cli.Config.Hosts[i]
 		proto, addr, ok := strings.Cut(protoAddr, "://")
 		if !ok || addr == "" {
 			return nil, fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
 		}
 
 		// It's a bad idea to bind to TCP without tlsverify.
-		authEnabled := serverConfig.TLSConfig != nil && serverConfig.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert
+		authEnabled := tlsConfig != nil && tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert
 		if proto == "tcp" && !authEnabled {
 			logrus.WithField("host", protoAddr).Warn("Binding to IP address without --tlsverify is insecure and gives root access on this machine to everyone who has access to your network.")
 			logrus.WithField("host", protoAddr).Warn("Binding to an IP address, even on localhost, can also give access to scripts run in a browser. Be safe out there!")
@@ -712,7 +704,7 @@ func loadListeners(cli *DaemonCli, serverConfig *apiserver.Config) ([]string, er
 				return nil, err
 			}
 		}
-		ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
+		ls, err := listeners.Init(proto, addr, cli.Config.SocketGroup, tlsConfig)
 		if err != nil {
 			return nil, err
 		}