Add otel support

This uses otel standard environment variables to configure tracing in
the daemon.
It also adds support for propagating trace contexts in the client and
reading those from the API server.

See
https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
for details on otel environment variables.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

api/server/router/grpc/grpc.go

@@ -4,49 +4,28 @@ import (
 	"context"
 	"strings"
 
-	"github.com/containerd/containerd/log"
 	"github.com/docker/docker/api/server/router"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/moby/buildkit/util/grpcerrors"
-	"github.com/moby/buildkit/util/tracing/detect"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	"go.opentelemetry.io/otel/propagation"
-	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/net/http2"
 	"google.golang.org/grpc"
 )
 
-func init() {
-	// enable in memory recording for grpc traces
-	detect.Recorder = detect.NewTraceRecorder()
-}
-
 type grpcRouter struct {
 	routes     []router.Route
 	grpcServer *grpc.Server
 	h2Server   *http2.Server
 }
 
-var propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
-
 // NewRouter initializes a new grpc http router
 func NewRouter(backends ...Backend) router.Router {
-	tp, err := detect.TracerProvider()
-	if err != nil {
-		log.G(context.TODO()).WithError(err).Error("failed to detect trace provider")
-	}
-
-	opts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor)}
-	if tp != nil {
-		streamTracer := otelgrpc.StreamServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators))
-		unary := grpc_middleware.ChainUnaryServer(unaryInterceptor(tp), grpcerrors.UnaryServerInterceptor)
-		stream := grpc_middleware.ChainStreamServer(streamTracer, grpcerrors.StreamServerInterceptor)
-		opts = []grpc.ServerOption{grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream)}
-	}
+	unary := grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptor(), grpcerrors.UnaryServerInterceptor))
+	stream := grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(otelgrpc.StreamServerInterceptor(), grpcerrors.StreamServerInterceptor))
 
 	r := &grpcRouter{
 		h2Server:   &http2.Server{},
-		grpcServer: grpc.NewServer(opts...),
+		grpcServer: grpc.NewServer(unary, stream),
 	}
 	for _, b := range backends {
 		b.RegisterGRPC(r.grpcServer)
@@ -66,8 +45,8 @@ func (gr *grpcRouter) initRoutes() {
 	}
 }
 
-func unaryInterceptor(tp trace.TracerProvider) grpc.UnaryServerInterceptor {
-	withTrace := otelgrpc.UnaryServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators))
+func unaryInterceptor() grpc.UnaryServerInterceptor {
+	withTrace := otelgrpc.UnaryServerInterceptor()
 
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
 		// This method is used by the clients to send their traces to buildkit so they can be included

api/server/server.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/api/server/router/debug"
 	"github.com/docker/docker/dockerversion"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
 // versionMatcher defines a variable matcher to be parsed by the router
@@ -30,7 +31,7 @@ func (s *Server) UseMiddleware(m middleware.Middleware) {
 }
 
 func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
+	return otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Define the context that we'll pass around to share info
 		// like the docker-request-id.
 		//
@@ -42,6 +43,7 @@ func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc {
 		// use intermediate variable to prevent "should not use basic type
 		// string as key in context.WithValue" golint errors
 		ctx := context.WithValue(r.Context(), dockerversion.UAStringKey{}, r.Header.Get("User-Agent"))
+
 		r = r.WithContext(ctx)
 		handlerFunc := s.handlerWithGlobalMiddlewares(handler)
 
@@ -57,7 +59,7 @@ func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc {
 			}
 			makeErrorHandler(err)(w, r)
 		}
-	}
+	}), "").ServeHTTP
 }
 
 type pageNotFoundError struct{}

builder/builder-next/controller.go

@@ -9,6 +9,7 @@ import (
 
 	ctd "github.com/containerd/containerd"
 	"github.com/containerd/containerd/content/local"
+	"github.com/containerd/containerd/log"
 	ctdmetadata "github.com/containerd/containerd/metadata"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/api/types"
@@ -43,12 +44,14 @@ import (
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/leaseutil"
 	"github.com/moby/buildkit/util/network/netproviders"
+	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/moby/buildkit/worker"
 	"github.com/moby/buildkit/worker/containerd"
 	"github.com/moby/buildkit/worker/label"
 	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 	bolt "go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/sdk/trace"
 
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
@@ -61,6 +64,14 @@ func newController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control
 	return newGraphDriverController(ctx, rt, opt)
 }
 
+func getTraceExporter(ctx context.Context) trace.SpanExporter {
+	exp, err := detect.Exporter()
+	if err != nil {
+		log.G(ctx).WithError(err).Error("Failed to detect trace exporter for buildkit controller")
+	}
+	return exp
+}
+
 func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 	if err := os.MkdirAll(opt.Root, 0o711); err != nil {
 		return nil, err
@@ -136,11 +147,12 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt
 			"local":    localremotecache.ResolveCacheExporterFunc(opt.SessionManager),
 			"registry": registryremotecache.ResolveCacheExporterFunc(opt.SessionManager, opt.RegistryHosts),
 		},
-		Entitlements:  getEntitlements(opt.BuilderConfig),
-		HistoryDB:     historyDB,
-		HistoryConfig: historyConf,
-		LeaseManager:  wo.LeaseManager,
-		ContentStore:  wo.ContentStore,
+		Entitlements:   getEntitlements(opt.BuilderConfig),
+		HistoryDB:      historyDB,
+		HistoryConfig:  historyConf,
+		LeaseManager:   wo.LeaseManager,
+		ContentStore:   wo.ContentStore,
+		TraceCollector: getTraceExporter(ctx),
 	})
 }
 
@@ -354,11 +366,12 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
 		ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
 			"inline": inlineremotecache.ResolveCacheExporterFunc(),
 		},
-		Entitlements:  getEntitlements(opt.BuilderConfig),
-		LeaseManager:  lm,
-		ContentStore:  store,
-		HistoryDB:     historyDB,
-		HistoryConfig: historyConf,
+		Entitlements:   getEntitlements(opt.BuilderConfig),
+		LeaseManager:   lm,
+		ContentStore:   store,
+		HistoryDB:      historyDB,
+		HistoryConfig:  historyConf,
+		TraceCollector: getTraceExporter(ctx),
 	})
 }
 

client/client.go

@@ -56,6 +56,8 @@ import (
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/go-connections/sockets"
 	"github.com/pkg/errors"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // DummyHost is a hostname used for local communication.
@@ -123,6 +125,12 @@ type Client struct {
 
 	// negotiated indicates that API version negotiation took place
 	negotiated bool
+
+	tp trace.TracerProvider
+
+	// When the client transport is an *http.Transport (default) we need to do some extra things (like closing idle connections).
+	// Store the original transport as the http.Client transport will be wrapped with tracing libs.
+	baseTransport *http.Transport
 }
 
 // ErrRedirect is the error returned by checkRedirect when the request is non-GET.
@@ -188,6 +196,12 @@ func NewClientWithOpts(ops ...Opt) (*Client, error) {
 		}
 	}
 
+	if tr, ok := c.client.Transport.(*http.Transport); ok {
+		// Store the base transport before we wrap it in tracing libs below
+		// This is used, as an example, to close idle connections when the client is closed
+		c.baseTransport = tr
+	}
+
 	if c.scheme == "" {
 		// TODO(stevvooe): This isn't really the right way to write clients in Go.
 		// `NewClient` should probably only take an `*http.Client` and work from there.
@@ -201,9 +215,24 @@ func NewClientWithOpts(ops ...Opt) (*Client, error) {
 		}
 	}
 
+	c.client.Transport = otelhttp.NewTransport(
+		c.client.Transport,
+		otelhttp.WithTracerProvider(c.tp),
+		otelhttp.WithSpanNameFormatter(func(_ string, req *http.Request) string {
+			return req.Method + " " + req.URL.Path
+		}),
+	)
+
 	return c, nil
 }
 
+func (cli *Client) tlsConfig() *tls.Config {
+	if cli.baseTransport == nil {
+		return nil
+	}
+	return cli.baseTransport.TLSClientConfig
+}
+
 func defaultHTTPClient(hostURL *url.URL) (*http.Client, error) {
 	transport := &http.Transport{}
 	err := sockets.ConfigureTransport(transport, hostURL.Scheme, hostURL.Host)
@@ -216,20 +245,11 @@ func defaultHTTPClient(hostURL *url.URL) (*http.Client, error) {
 	}, nil
 }
 
-// tlsConfig returns the TLS configuration from the client's transport.
-// It returns nil if the transport is not a [http.Transport], or if no
-// TLSClientConfig is set.
-func (cli *Client) tlsConfig() *tls.Config {
-	if tr, ok := cli.client.Transport.(*http.Transport); ok {
-		return tr.TLSClientConfig
-	}
-	return nil
-}
-
 // Close the transport used by the client
 func (cli *Client) Close() error {
-	if t, ok := cli.client.Transport.(*http.Transport); ok {
-		t.CloseIdleConnections()
+	if cli.baseTransport != nil {
+		cli.baseTransport.CloseIdleConnections()
+		return nil
 	}
 	return nil
 }
@@ -356,6 +376,20 @@ func ParseHostURL(host string) (*url.URL, error) {
 	}, nil
 }
 
+func (cli *Client) dialerFromTransport() func(context.Context, string, string) (net.Conn, error) {
+	if cli.baseTransport == nil || cli.baseTransport.DialContext == nil {
+		return nil
+	}
+
+	if cli.baseTransport.TLSClientConfig != nil {
+		// When using a tls config we don't use the configured dialer but instead a fallback dialer...
+		// Note: It seems like this should use the normal dialer and wrap the returned net.Conn in a tls.Conn
+		// I honestly don't know why it doesn't do that, but it doesn't and such a change is entirely unrelated to the change in this commit.
+		return nil
+	}
+	return cli.baseTransport.DialContext
+}
+
 // Dialer returns a dialer for a raw stream connection, with an HTTP/1.1 header,
 // that can be used for proxying the daemon connection. It is used by
 // ["docker dial-stdio"].
@@ -363,10 +397,8 @@ func ParseHostURL(host string) (*url.URL, error) {
 // ["docker dial-stdio"]: https://github.com/docker/cli/pull/1014
 func (cli *Client) Dialer() func(context.Context) (net.Conn, error) {
 	return func(ctx context.Context) (net.Conn, error) {
-		if transport, ok := cli.client.Transport.(*http.Transport); ok {
-			if transport.DialContext != nil && transport.TLSClientConfig == nil {
-				return transport.DialContext(ctx, cli.proto, cli.addr)
-			}
+		if dialFn := cli.dialerFromTransport(); dialFn != nil {
+			return dialFn(ctx, cli.proto, cli.addr)
 		}
 		switch cli.proto {
 		case "unix":

client/client_mock_test.go

@@ -17,9 +17,21 @@ func (tf transportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
 	return tf(req)
 }
 
+func transportEnsureBody(f transportFunc) transportFunc {
+	return func(req *http.Request) (*http.Response, error) {
+		resp, err := f(req)
+		if resp != nil && resp.Body == nil {
+			resp.Body = http.NoBody
+		}
+		return resp, err
+	}
+}
+
 func newMockClient(doer func(*http.Request) (*http.Response, error)) *http.Client {
 	return &http.Client{
-		Transport: transportFunc(doer),
+		// Some tests return a response with a nil body, this is incorrect semantically and causes a panic with wrapper transports (such as otelhttp's)
+		// Wrap the doer to ensure a body is always present even if it is empty.
+		Transport: transportEnsureBody(transportFunc(doer)),
 	}
 }
 

client/client_test.go

@@ -101,9 +101,9 @@ func TestNewClientWithOpsFromEnv(t *testing.T) {
 
 			if tc.envs["DOCKER_TLS_VERIFY"] != "" {
 				// pedantic checking that this is handled correctly
-				tr := client.client.Transport.(*http.Transport)
-				assert.Assert(t, tr.TLSClientConfig != nil)
-				assert.Check(t, is.Equal(tr.TLSClientConfig.InsecureSkipVerify, false))
+				tlsConfig := client.tlsConfig()
+				assert.Assert(t, tlsConfig != nil)
+				assert.Check(t, is.Equal(tlsConfig.InsecureSkipVerify, false))
 			}
 		})
 	}

client/hijack.go

@@ -13,6 +13,11 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // postHijacked sends a POST request and hijacks the connection.
@@ -45,11 +50,32 @@ func (cli *Client) DialHijack(ctx context.Context, url, proto string, meta map[s
 	return conn, err
 }
 
-func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, string, error) {
+func (cli *Client) setupHijackConn(req *http.Request, proto string) (_ net.Conn, _ string, retErr error) {
 	ctx := req.Context()
 	req.Header.Set("Connection", "Upgrade")
 	req.Header.Set("Upgrade", proto)
 
+	// We aren't using the configured RoundTripper here so manually inject the trace context
+	tp := cli.tp
+	if tp == nil {
+		if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
+			tp = span.TracerProvider()
+		} else {
+			tp = otel.GetTracerProvider()
+		}
+	}
+
+	ctx, span := tp.Tracer("").Start(ctx, req.Method+" "+req.URL.Path)
+	span.SetAttributes(semconv.HTTPClientAttributesFromHTTPRequest(req)...)
+	defer func() {
+		if retErr != nil {
+			span.RecordError(retErr)
+			span.SetStatus(codes.Error, retErr.Error())
+		}
+		span.End()
+	}()
+	otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header))
+
 	dialer := cli.Dialer()
 	conn, err := dialer(ctx)
 	if err != nil {
@@ -71,6 +97,9 @@ func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, s
 
 	// Server hijacks the connection, error 'connection closed' expected
 	resp, err := clientconn.Do(req)
+	if resp != nil {
+		span.SetStatus(semconv.SpanStatusFromHTTPStatusCode(resp.StatusCode))
+	}
 
 	//nolint:staticcheck // ignore SA1019 for connecting to old (pre go1.8) daemons
 	if err != httputil.ErrPersistEOF {

client/options.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/go-connections/sockets"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // Opt is a configuration option to initialize a [Client].
@@ -221,3 +222,12 @@ func WithAPIVersionNegotiation() Opt {
 		return nil
 	}
 }
+
+// WithTraceProvider sets the trace provider for the client.
+// If this is not set then the global trace provider will be used.
+func WithTraceProvider(provider trace.TracerProvider) Opt {
+	return func(c *Client) error {
+		c.tp = provider
+		return nil
+	}
+}

cmd/dockerd/daemon.go

@@ -17,6 +17,7 @@ import (
 	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	containerddefaults "github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/tracing"
 	"github.com/docker/docker/api"
 	apiserver "github.com/docker/docker/api/server"
 	buildbackend "github.com/docker/docker/api/server/backend/build"
@@ -56,10 +57,14 @@ import (
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/util/bklog"
+	"github.com/moby/buildkit/util/tracing/detect"
 	swarmapi "github.com/moby/swarmkit/v2/api"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
 )
 
 // DaemonCli represents the daemon CLI.
@@ -227,6 +232,24 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 	// Notify that the API is active, but before daemon is set up.
 	preNotifyReady()
 
+	const otelServiceNameEnv = "OTEL_SERVICE_NAME"
+	if _, ok := os.LookupEnv(otelServiceNameEnv); !ok {
+		os.Setenv(otelServiceNameEnv, filepath.Base(os.Args[0]))
+	}
+
+	setOTLPProtoDefault()
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
+	detect.Recorder = detect.NewTraceRecorder()
+
+	tp, err := detect.TracerProvider()
+	if err != nil {
+		log.G(ctx).WithError(err).Warn("Failed to initialize tracing, skipping")
+	} else {
+		otel.SetTracerProvider(tp)
+		log.G(ctx).Logger.AddHook(tracing.NewLogrusHook())
+		bklog.G(ctx).Logger.AddHook(tracing.NewLogrusHook())
+	}
+
 	pluginStore := plugin.NewStore()
 
 	var apiServer apiserver.Server
@@ -279,6 +302,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 	if err != nil {
 		return err
 	}
+
 	routerOptions.cluster = c
 
 	httpServer.Handler = apiServer.CreateMux(routerOptions.Build()...)
@@ -330,10 +354,27 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return errors.Wrap(err, "shutting down due to ServeAPI error")
 	}
 
+	detect.Shutdown(context.Background())
+
 	log.G(ctx).Info("Daemon shutdown complete")
 	return nil
 }
 
+// The buildkit "detect" package uses grpc as the default proto, which is in conformance with the old spec.
+// For a little while now http/protobuf is the default spec, so this function sets the protocol to http/protobuf when the env var is unset
+// so that the detect package will use http/protobuf as a default.
+// TODO: This can be removed after buildkit is updated to use http/protobuf as the default.
+func setOTLPProtoDefault() {
+	const (
+		tracesEnv = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"
+		protoEnv  = "OTEL_EXPORTER_OTLP_PROTOCOL"
+	)
+
+	if os.Getenv(tracesEnv) == "" && os.Getenv(protoEnv) == "" {
+		os.Setenv(tracesEnv, "http/protobuf")
+	}
+}
+
 type routerOptions struct {
 	sessionManager *session.Manager
 	buildBackend   *buildbackend.Backend

daemon/daemon.go

@@ -71,6 +71,7 @@ import (
 	"github.com/moby/locker"
 	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
@@ -935,10 +936,17 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		// TODO(stevvooe): We may need to allow configuration of this on the client.
 		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+		grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
+		grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
 	}
 
 	if cfgStore.ContainerdAddr != "" {
-		d.containerdClient, err = containerd.New(cfgStore.ContainerdAddr, containerd.WithDefaultNamespace(cfgStore.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+		d.containerdClient, err = containerd.New(
+			cfgStore.ContainerdAddr,
+			containerd.WithDefaultNamespace(cfgStore.ContainerdNamespace),
+			containerd.WithDialOpts(gopts),
+			containerd.WithTimeout(60*time.Second),
+		)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to dial %q", cfgStore.ContainerdAddr)
 		}
@@ -948,7 +956,12 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		var pluginCli *containerd.Client
 
 		if cfgStore.ContainerdAddr != "" {
-			pluginCli, err = containerd.New(cfgStore.ContainerdAddr, containerd.WithDefaultNamespace(cfgStore.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+			pluginCli, err = containerd.New(
+				cfgStore.ContainerdAddr,
+				containerd.WithDefaultNamespace(cfgStore.ContainerdPluginNamespace),
+				containerd.WithDialOpts(gopts),
+				containerd.WithTimeout(60*time.Second),
+			)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to dial %q", cfgStore.ContainerdAddr)
 			}
@@ -1005,7 +1018,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, errors.New("Devices cgroup isn't mounted")
 	}
 
-	d.id, err = loadOrCreateID(filepath.Join(cfgStore.Root, "engine-id"))
+	d.id, err = LoadOrCreateID(cfgStore.Root)
 	if err != nil {
 		return nil, err
 	}

daemon/id.go

@@ -2,21 +2,25 @@ package daemon // import "github.com/docker/docker/daemon"
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
-// loadOrCreateID loads the engine's ID from idPath, or generates a new ID
+const idFilename = "engine-id"
+
+// LoadOrCreateID loads the engine's ID from the given root, or generates a new ID
 // if it doesn't exist. It returns the ID, and any error that occurred when
 // saving the file.
 //
 // Note that this function expects the daemon's root directory to already have
 // been created with the right permissions and ownership (usually this would
 // be done by daemon.CreateDaemonRoot().
-func loadOrCreateID(idPath string) (string, error) {
+func LoadOrCreateID(root string) (string, error) {
 	var id string
+	idPath := filepath.Join(root, idFilename)
 	idb, err := os.ReadFile(idPath)
 	if os.IsNotExist(err) {
 		id = uuid.New().String()

integration/container/pidmode_linux_test.go

@@ -40,9 +40,8 @@ func TestPIDModeHost(t *testing.T) {
 func TestPIDModeContainer(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 
-	defer setupTest(t)()
+	ctx := setupTest(t)
 	apiClient := testEnv.APIClient()
-	ctx := context.Background()
 
 	t.Run("non-existing container", func(t *testing.T) {
 		_, err := container.CreateFromConfig(ctx, apiClient, container.NewTestConfig(container.WithPIDMode("container:nosuchcontainer")))

integration/image/remove_test.go

@@ -63,8 +63,7 @@ func TestRemoveImageOrphaning(t *testing.T) {
 func TestRemoveByDigest(t *testing.T) {
 	skip.If(t, !testEnv.UsingSnapshotter(), "RepoDigests doesn't include tags when using graphdrivers")
 
-	defer setupTest(t)()
-	ctx := context.Background()
+	ctx := setupTest(t)
 	client := testEnv.APIClient()
 
 	err := client.ImageTag(ctx, "busybox", "test-remove-by-digest:latest")

libcontainerd/supervisor/remote_daemon.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/services/server/config"
 	"github.com/containerd/containerd/sys"
@@ -19,6 +20,9 @@ import (
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 const (
@@ -295,7 +299,17 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 				continue
 			}
 
-			client, err = containerd.New(r.GRPC.Address, containerd.WithTimeout(60*time.Second))
+			client, err = containerd.New(
+				r.GRPC.Address,
+				containerd.WithTimeout(60*time.Second),
+				containerd.WithDialOpts([]grpc.DialOption{
+					grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
+					grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
+					grpc.WithTransportCredentials(insecure.NewCredentials()),
+					grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+					grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+				}),
+			)
 			if err != nil {
 				r.logger.WithError(err).Error("failed connecting to containerd")
 				delay = 100 * time.Millisecond

vendor.mod

@@ -92,7 +92,12 @@ require (
 	github.com/vishvananda/netns v0.0.2
 	go.etcd.io/bbolt v1.3.7
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0
 	go.opentelemetry.io/otel v1.4.1
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.4.1 // indirect
+	go.opentelemetry.io/otel/sdk v1.4.1
 	go.opentelemetry.io/otel/trace v1.4.1
 	golang.org/x/mod v0.10.0
 	golang.org/x/net v0.10.0
@@ -187,14 +192,9 @@ require (
 	go.etcd.io/etcd/server/v3 v3.5.6 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.29.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.1 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.4.1 // indirect
 	go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
 	go.opentelemetry.io/otel/metric v0.27.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.4.1 // indirect
 	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect

vendor/github.com/containerd/containerd/tracing/log.go

@@ -0,0 +1,130 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package tracing
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// NewLogrusHook creates a new logrus hook
+func NewLogrusHook() *LogrusHook {
+	return &LogrusHook{}
+}
+
+// LogrusHook is a logrus hook which adds logrus events to active spans.
+// If the span is not recording or the span context is invalid, the hook is a no-op.
+type LogrusHook struct{}
+
+// Levels returns the logrus levels that this hook is interested in.
+func (h *LogrusHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}
+
+// Fire is called when a log event occurs.
+func (h *LogrusHook) Fire(entry *logrus.Entry) error {
+	span := trace.SpanFromContext(entry.Context)
+	if span == nil {
+		return nil
+	}
+
+	if !span.SpanContext().IsValid() || !span.IsRecording() {
+		return nil
+	}
+
+	span.AddEvent(
+		entry.Message,
+		trace.WithAttributes(logrusDataToAttrs(entry.Data)...),
+		trace.WithAttributes(attribute.String("level", entry.Level.String())),
+		trace.WithTimestamp(entry.Time),
+	)
+
+	return nil
+}
+
+func logrusDataToAttrs(data logrus.Fields) []attribute.KeyValue {
+	attrs := make([]attribute.KeyValue, 0, len(data))
+	for k, v := range data {
+		attrs = append(attrs, any(k, v))
+	}
+	return attrs
+}
+
+func any(k string, v interface{}) attribute.KeyValue {
+	if v == nil {
+		return attribute.String(k, "<nil>")
+	}
+
+	switch typed := v.(type) {
+	case bool:
+		return attribute.Bool(k, typed)
+	case []bool:
+		return attribute.BoolSlice(k, typed)
+	case int:
+		return attribute.Int(k, typed)
+	case []int:
+		return attribute.IntSlice(k, typed)
+	case int8:
+		return attribute.Int(k, int(typed))
+	case []int8:
+		ls := make([]int, 0, len(typed))
+		for _, i := range typed {
+			ls = append(ls, int(i))
+		}
+		return attribute.IntSlice(k, ls)
+	case int16:
+		return attribute.Int(k, int(typed))
+	case []int16:
+		ls := make([]int, 0, len(typed))
+		for _, i := range typed {
+			ls = append(ls, int(i))
+		}
+		return attribute.IntSlice(k, ls)
+	case int32:
+		return attribute.Int64(k, int64(typed))
+	case []int32:
+		ls := make([]int64, 0, len(typed))
+		for _, i := range typed {
+			ls = append(ls, int64(i))
+		}
+		return attribute.Int64Slice(k, ls)
+	case int64:
+		return attribute.Int64(k, typed)
+	case []int64:
+		return attribute.Int64Slice(k, typed)
+	case float64:
+		return attribute.Float64(k, typed)
+	case []float64:
+		return attribute.Float64Slice(k, typed)
+	case string:
+		return attribute.String(k, typed)
+	case []string:
+		return attribute.StringSlice(k, typed)
+	}
+
+	if stringer, ok := v.(fmt.Stringer); ok {
+		return attribute.String(k, stringer.String())
+	}
+	if b, err := json.Marshal(v); b != nil && err == nil {
+		return attribute.String(k, string(b))
+	}
+	return attribute.String(k, fmt.Sprintf("%v", v))
+}

vendor/github.com/containerd/containerd/tracing/tracing.go

@@ -0,0 +1,37 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package tracing
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// StartSpan starts child span in a context.
+func StartSpan(ctx context.Context, opName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	if parent := trace.SpanFromContext(ctx); parent != nil && parent.SpanContext().IsValid() {
+		return parent.TracerProvider().Tracer("").Start(ctx, opName, opts...)
+	}
+	return otel.Tracer("").Start(ctx, opName, opts...)
+}
+
+// StopSpan ends the span specified
+func StopSpan(span trace.Span) {
+	span.End()
+}

vendor/modules.txt

@@ -319,6 +319,7 @@ github.com/containerd/containerd/snapshots/overlay/overlayutils
 github.com/containerd/containerd/snapshots/proxy
 github.com/containerd/containerd/sys
 github.com/containerd/containerd/sys/reaper
+github.com/containerd/containerd/tracing
 github.com/containerd/containerd/version
 # github.com/containerd/continuity v0.4.1
 ## explicit; go 1.19