From 642e9917ff1df47b8672084c0d8869c68c34cf1e Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 14 Jul 2023 17:59:45 +0000 Subject: [PATCH] Add otel support This uses otel standard environment variables to configure tracing in the daemon. It also adds support for propagating trace contexts in the client and reading those from the API server. See https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ for details on otel environment variables. Signed-off-by: Brian Goff --- api/server/router/grpc/grpc.go | 31 +---- api/server/server.go | 6 +- builder/builder-next/controller.go | 33 +++-- client/client.go | 64 ++++++--- client/client_mock_test.go | 14 +- client/client_test.go | 6 +- client/hijack.go | 31 ++++- client/options.go | 10 ++ cmd/dockerd/daemon.go | 41 ++++++ daemon/daemon.go | 19 ++- daemon/id.go | 8 +- integration/container/pidmode_linux_test.go | 3 +- integration/image/remove_test.go | 3 +- libcontainerd/supervisor/remote_daemon.go | 16 ++- vendor.mod | 10 +- .../containerd/containerd/tracing/log.go | 130 ++++++++++++++++++ .../containerd/containerd/tracing/tracing.go | 37 +++++ vendor/modules.txt | 1 + 18 files changed, 389 insertions(+), 74 deletions(-) create mode 100644 vendor/github.com/containerd/containerd/tracing/log.go create mode 100644 vendor/github.com/containerd/containerd/tracing/tracing.go diff --git a/api/server/router/grpc/grpc.go b/api/server/router/grpc/grpc.go index 041fc3529b..4ea183192d 100644 --- a/api/server/router/grpc/grpc.go +++ b/api/server/router/grpc/grpc.go @@ -4,49 +4,28 @@ import ( "context" "strings" - "github.com/containerd/containerd/log" "github.com/docker/docker/api/server/router" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/moby/buildkit/util/grpcerrors" - "github.com/moby/buildkit/util/tracing/detect" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/trace" "golang.org/x/net/http2" "google.golang.org/grpc" ) -func init() { - // enable in memory recording for grpc traces - detect.Recorder = detect.NewTraceRecorder() -} - type grpcRouter struct { routes []router.Route grpcServer *grpc.Server h2Server *http2.Server } -var propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) - // NewRouter initializes a new grpc http router func NewRouter(backends ...Backend) router.Router { - tp, err := detect.TracerProvider() - if err != nil { - log.G(context.TODO()).WithError(err).Error("failed to detect trace provider") - } - - opts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor)} - if tp != nil { - streamTracer := otelgrpc.StreamServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators)) - unary := grpc_middleware.ChainUnaryServer(unaryInterceptor(tp), grpcerrors.UnaryServerInterceptor) - stream := grpc_middleware.ChainStreamServer(streamTracer, grpcerrors.StreamServerInterceptor) - opts = []grpc.ServerOption{grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream)} - } + unary := grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptor(), grpcerrors.UnaryServerInterceptor)) + stream := grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(otelgrpc.StreamServerInterceptor(), grpcerrors.StreamServerInterceptor)) r := &grpcRouter{ h2Server: &http2.Server{}, - grpcServer: grpc.NewServer(opts...), + grpcServer: grpc.NewServer(unary, stream), } for _, b := range backends { b.RegisterGRPC(r.grpcServer) @@ -66,8 +45,8 @@ func (gr *grpcRouter) initRoutes() { } } -func unaryInterceptor(tp trace.TracerProvider) grpc.UnaryServerInterceptor { - withTrace := otelgrpc.UnaryServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators)) +func unaryInterceptor() grpc.UnaryServerInterceptor { + withTrace := otelgrpc.UnaryServerInterceptor() return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { // This method is used by the clients to send their traces to buildkit so they can be included diff --git a/api/server/server.go b/api/server/server.go index c1abfbee51..7d62dd504a 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/docker/docker/api/server/router/debug" "github.com/docker/docker/dockerversion" "github.com/gorilla/mux" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) // versionMatcher defines a variable matcher to be parsed by the router @@ -30,7 +31,7 @@ func (s *Server) UseMiddleware(m middleware.Middleware) { } func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { + return otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Define the context that we'll pass around to share info // like the docker-request-id. // @@ -42,6 +43,7 @@ func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc { // use intermediate variable to prevent "should not use basic type // string as key in context.WithValue" golint errors ctx := context.WithValue(r.Context(), dockerversion.UAStringKey{}, r.Header.Get("User-Agent")) + r = r.WithContext(ctx) handlerFunc := s.handlerWithGlobalMiddlewares(handler) @@ -57,7 +59,7 @@ func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc { } makeErrorHandler(err)(w, r) } - } + }), "").ServeHTTP } type pageNotFoundError struct{} diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index 3087697e92..b9805f7ebd 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -9,6 +9,7 @@ import ( ctd "github.com/containerd/containerd" "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/log" ctdmetadata "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/snapshots" "github.com/docker/docker/api/types" @@ -43,12 +44,14 @@ import ( "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/network/netproviders" + "github.com/moby/buildkit/util/tracing/detect" "github.com/moby/buildkit/worker" "github.com/moby/buildkit/worker/containerd" "github.com/moby/buildkit/worker/label" "github.com/pkg/errors" "go.etcd.io/bbolt" bolt "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/sdk/trace" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" @@ -61,6 +64,14 @@ func newController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control return newGraphDriverController(ctx, rt, opt) } +func getTraceExporter(ctx context.Context) trace.SpanExporter { + exp, err := detect.Exporter() + if err != nil { + log.G(ctx).WithError(err).Error("Failed to detect trace exporter for buildkit controller") + } + return exp +} + func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { if err := os.MkdirAll(opt.Root, 0o711); err != nil { return nil, err @@ -136,11 +147,12 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt "local": localremotecache.ResolveCacheExporterFunc(opt.SessionManager), "registry": registryremotecache.ResolveCacheExporterFunc(opt.SessionManager, opt.RegistryHosts), }, - Entitlements: getEntitlements(opt.BuilderConfig), - HistoryDB: historyDB, - HistoryConfig: historyConf, - LeaseManager: wo.LeaseManager, - ContentStore: wo.ContentStore, + Entitlements: getEntitlements(opt.BuilderConfig), + HistoryDB: historyDB, + HistoryConfig: historyConf, + LeaseManager: wo.LeaseManager, + ContentStore: wo.ContentStore, + TraceCollector: getTraceExporter(ctx), }) } @@ -354,11 +366,12 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ "inline": inlineremotecache.ResolveCacheExporterFunc(), }, - Entitlements: getEntitlements(opt.BuilderConfig), - LeaseManager: lm, - ContentStore: store, - HistoryDB: historyDB, - HistoryConfig: historyConf, + Entitlements: getEntitlements(opt.BuilderConfig), + LeaseManager: lm, + ContentStore: store, + HistoryDB: historyDB, + HistoryConfig: historyConf, + TraceCollector: getTraceExporter(ctx), }) } diff --git a/client/client.go b/client/client.go index fd357a93e2..c321e5198b 100644 --- a/client/client.go +++ b/client/client.go @@ -56,6 +56,8 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/docker/go-connections/sockets" "github.com/pkg/errors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/trace" ) // DummyHost is a hostname used for local communication. @@ -123,6 +125,12 @@ type Client struct { // negotiated indicates that API version negotiation took place negotiated bool + + tp trace.TracerProvider + + // When the client transport is an *http.Transport (default) we need to do some extra things (like closing idle connections). + // Store the original transport as the http.Client transport will be wrapped with tracing libs. + baseTransport *http.Transport } // ErrRedirect is the error returned by checkRedirect when the request is non-GET. @@ -188,6 +196,12 @@ func NewClientWithOpts(ops ...Opt) (*Client, error) { } } + if tr, ok := c.client.Transport.(*http.Transport); ok { + // Store the base transport before we wrap it in tracing libs below + // This is used, as an example, to close idle connections when the client is closed + c.baseTransport = tr + } + if c.scheme == "" { // TODO(stevvooe): This isn't really the right way to write clients in Go. // `NewClient` should probably only take an `*http.Client` and work from there. @@ -201,9 +215,24 @@ func NewClientWithOpts(ops ...Opt) (*Client, error) { } } + c.client.Transport = otelhttp.NewTransport( + c.client.Transport, + otelhttp.WithTracerProvider(c.tp), + otelhttp.WithSpanNameFormatter(func(_ string, req *http.Request) string { + return req.Method + " " + req.URL.Path + }), + ) + return c, nil } +func (cli *Client) tlsConfig() *tls.Config { + if cli.baseTransport == nil { + return nil + } + return cli.baseTransport.TLSClientConfig +} + func defaultHTTPClient(hostURL *url.URL) (*http.Client, error) { transport := &http.Transport{} err := sockets.ConfigureTransport(transport, hostURL.Scheme, hostURL.Host) @@ -216,20 +245,11 @@ func defaultHTTPClient(hostURL *url.URL) (*http.Client, error) { }, nil } -// tlsConfig returns the TLS configuration from the client's transport. -// It returns nil if the transport is not a [http.Transport], or if no -// TLSClientConfig is set. -func (cli *Client) tlsConfig() *tls.Config { - if tr, ok := cli.client.Transport.(*http.Transport); ok { - return tr.TLSClientConfig - } - return nil -} - // Close the transport used by the client func (cli *Client) Close() error { - if t, ok := cli.client.Transport.(*http.Transport); ok { - t.CloseIdleConnections() + if cli.baseTransport != nil { + cli.baseTransport.CloseIdleConnections() + return nil } return nil } @@ -356,6 +376,20 @@ func ParseHostURL(host string) (*url.URL, error) { }, nil } +func (cli *Client) dialerFromTransport() func(context.Context, string, string) (net.Conn, error) { + if cli.baseTransport == nil || cli.baseTransport.DialContext == nil { + return nil + } + + if cli.baseTransport.TLSClientConfig != nil { + // When using a tls config we don't use the configured dialer but instead a fallback dialer... + // Note: It seems like this should use the normal dialer and wrap the returned net.Conn in a tls.Conn + // I honestly don't know why it doesn't do that, but it doesn't and such a change is entirely unrelated to the change in this commit. + return nil + } + return cli.baseTransport.DialContext +} + // Dialer returns a dialer for a raw stream connection, with an HTTP/1.1 header, // that can be used for proxying the daemon connection. It is used by // ["docker dial-stdio"]. @@ -363,10 +397,8 @@ func ParseHostURL(host string) (*url.URL, error) { // ["docker dial-stdio"]: https://github.com/docker/cli/pull/1014 func (cli *Client) Dialer() func(context.Context) (net.Conn, error) { return func(ctx context.Context) (net.Conn, error) { - if transport, ok := cli.client.Transport.(*http.Transport); ok { - if transport.DialContext != nil && transport.TLSClientConfig == nil { - return transport.DialContext(ctx, cli.proto, cli.addr) - } + if dialFn := cli.dialerFromTransport(); dialFn != nil { + return dialFn(ctx, cli.proto, cli.addr) } switch cli.proto { case "unix": diff --git a/client/client_mock_test.go b/client/client_mock_test.go index c119e59bbb..4c9989f25f 100644 --- a/client/client_mock_test.go +++ b/client/client_mock_test.go @@ -17,9 +17,21 @@ func (tf transportFunc) RoundTrip(req *http.Request) (*http.Response, error) { return tf(req) } +func transportEnsureBody(f transportFunc) transportFunc { + return func(req *http.Request) (*http.Response, error) { + resp, err := f(req) + if resp != nil && resp.Body == nil { + resp.Body = http.NoBody + } + return resp, err + } +} + func newMockClient(doer func(*http.Request) (*http.Response, error)) *http.Client { return &http.Client{ - Transport: transportFunc(doer), + // Some tests return a response with a nil body, this is incorrect semantically and causes a panic with wrapper transports (such as otelhttp's) + // Wrap the doer to ensure a body is always present even if it is empty. + Transport: transportEnsureBody(transportFunc(doer)), } } diff --git a/client/client_test.go b/client/client_test.go index 5d74b193ea..e653c16fc8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -101,9 +101,9 @@ func TestNewClientWithOpsFromEnv(t *testing.T) { if tc.envs["DOCKER_TLS_VERIFY"] != "" { // pedantic checking that this is handled correctly - tr := client.client.Transport.(*http.Transport) - assert.Assert(t, tr.TLSClientConfig != nil) - assert.Check(t, is.Equal(tr.TLSClientConfig.InsecureSkipVerify, false)) + tlsConfig := client.tlsConfig() + assert.Assert(t, tlsConfig != nil) + assert.Check(t, is.Equal(tlsConfig.InsecureSkipVerify, false)) } }) } diff --git a/client/hijack.go b/client/hijack.go index 3fc17dc071..9e7f6a66c3 100644 --- a/client/hijack.go +++ b/client/hijack.go @@ -13,6 +13,11 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" + "go.opentelemetry.io/otel/trace" ) // postHijacked sends a POST request and hijacks the connection. @@ -45,11 +50,32 @@ func (cli *Client) DialHijack(ctx context.Context, url, proto string, meta map[s return conn, err } -func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, string, error) { +func (cli *Client) setupHijackConn(req *http.Request, proto string) (_ net.Conn, _ string, retErr error) { ctx := req.Context() req.Header.Set("Connection", "Upgrade") req.Header.Set("Upgrade", proto) + // We aren't using the configured RoundTripper here so manually inject the trace context + tp := cli.tp + if tp == nil { + if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { + tp = span.TracerProvider() + } else { + tp = otel.GetTracerProvider() + } + } + + ctx, span := tp.Tracer("").Start(ctx, req.Method+" "+req.URL.Path) + span.SetAttributes(semconv.HTTPClientAttributesFromHTTPRequest(req)...) + defer func() { + if retErr != nil { + span.RecordError(retErr) + span.SetStatus(codes.Error, retErr.Error()) + } + span.End() + }() + otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header)) + dialer := cli.Dialer() conn, err := dialer(ctx) if err != nil { @@ -71,6 +97,9 @@ func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, s // Server hijacks the connection, error 'connection closed' expected resp, err := clientconn.Do(req) + if resp != nil { + span.SetStatus(semconv.SpanStatusFromHTTPStatusCode(resp.StatusCode)) + } //nolint:staticcheck // ignore SA1019 for connecting to old (pre go1.8) daemons if err != httputil.ErrPersistEOF { diff --git a/client/options.go b/client/options.go index 5f9c4099ea..ddb0ca3991 100644 --- a/client/options.go +++ b/client/options.go @@ -11,6 +11,7 @@ import ( "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" "github.com/pkg/errors" + "go.opentelemetry.io/otel/trace" ) // Opt is a configuration option to initialize a [Client]. @@ -221,3 +222,12 @@ func WithAPIVersionNegotiation() Opt { return nil } } + +// WithTraceProvider sets the trace provider for the client. +// If this is not set then the global trace provider will be used. +func WithTraceProvider(provider trace.TracerProvider) Opt { + return func(c *Client) error { + c.tp = provider + return nil + } +} diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index 267007792a..41a4fc71e5 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -17,6 +17,7 @@ import ( "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" containerddefaults "github.com/containerd/containerd/defaults" "github.com/containerd/containerd/log" + "github.com/containerd/containerd/tracing" "github.com/docker/docker/api" apiserver "github.com/docker/docker/api/server" buildbackend "github.com/docker/docker/api/server/backend/build" @@ -56,10 +57,14 @@ import ( "github.com/docker/docker/runconfig" "github.com/docker/go-connections/tlsconfig" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/tracing/detect" swarmapi "github.com/moby/swarmkit/v2/api" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/pflag" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) // DaemonCli represents the daemon CLI. @@ -227,6 +232,24 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { // Notify that the API is active, but before daemon is set up. preNotifyReady() + const otelServiceNameEnv = "OTEL_SERVICE_NAME" + if _, ok := os.LookupEnv(otelServiceNameEnv); !ok { + os.Setenv(otelServiceNameEnv, filepath.Base(os.Args[0])) + } + + setOTLPProtoDefault() + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + detect.Recorder = detect.NewTraceRecorder() + + tp, err := detect.TracerProvider() + if err != nil { + log.G(ctx).WithError(err).Warn("Failed to initialize tracing, skipping") + } else { + otel.SetTracerProvider(tp) + log.G(ctx).Logger.AddHook(tracing.NewLogrusHook()) + bklog.G(ctx).Logger.AddHook(tracing.NewLogrusHook()) + } + pluginStore := plugin.NewStore() var apiServer apiserver.Server @@ -279,6 +302,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { if err != nil { return err } + routerOptions.cluster = c httpServer.Handler = apiServer.CreateMux(routerOptions.Build()...) @@ -330,10 +354,27 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { return errors.Wrap(err, "shutting down due to ServeAPI error") } + detect.Shutdown(context.Background()) + log.G(ctx).Info("Daemon shutdown complete") return nil } +// The buildkit "detect" package uses grpc as the default proto, which is in conformance with the old spec. +// For a little while now http/protobuf is the default spec, so this function sets the protocol to http/protobuf when the env var is unset +// so that the detect package will use http/protobuf as a default. +// TODO: This can be removed after buildkit is updated to use http/protobuf as the default. +func setOTLPProtoDefault() { + const ( + tracesEnv = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL" + protoEnv = "OTEL_EXPORTER_OTLP_PROTOCOL" + ) + + if os.Getenv(tracesEnv) == "" && os.Getenv(protoEnv) == "" { + os.Setenv(tracesEnv, "http/protobuf") + } +} + type routerOptions struct { sessionManager *session.Manager buildBackend *buildbackend.Backend diff --git a/daemon/daemon.go b/daemon/daemon.go index ca09cc7359..f93788d692 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -71,6 +71,7 @@ import ( "github.com/moby/locker" "github.com/pkg/errors" "go.etcd.io/bbolt" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "golang.org/x/sync/semaphore" "google.golang.org/grpc" "google.golang.org/grpc/backoff" @@ -935,10 +936,17 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S // TODO(stevvooe): We may need to allow configuration of this on the client. grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), + grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()), } if cfgStore.ContainerdAddr != "" { - d.containerdClient, err = containerd.New(cfgStore.ContainerdAddr, containerd.WithDefaultNamespace(cfgStore.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) + d.containerdClient, err = containerd.New( + cfgStore.ContainerdAddr, + containerd.WithDefaultNamespace(cfgStore.ContainerdNamespace), + containerd.WithDialOpts(gopts), + containerd.WithTimeout(60*time.Second), + ) if err != nil { return nil, errors.Wrapf(err, "failed to dial %q", cfgStore.ContainerdAddr) } @@ -948,7 +956,12 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S var pluginCli *containerd.Client if cfgStore.ContainerdAddr != "" { - pluginCli, err = containerd.New(cfgStore.ContainerdAddr, containerd.WithDefaultNamespace(cfgStore.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second)) + pluginCli, err = containerd.New( + cfgStore.ContainerdAddr, + containerd.WithDefaultNamespace(cfgStore.ContainerdPluginNamespace), + containerd.WithDialOpts(gopts), + containerd.WithTimeout(60*time.Second), + ) if err != nil { return nil, errors.Wrapf(err, "failed to dial %q", cfgStore.ContainerdAddr) } @@ -1005,7 +1018,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, errors.New("Devices cgroup isn't mounted") } - d.id, err = loadOrCreateID(filepath.Join(cfgStore.Root, "engine-id")) + d.id, err = LoadOrCreateID(cfgStore.Root) if err != nil { return nil, err } diff --git a/daemon/id.go b/daemon/id.go index d684dca357..92492eb31e 100644 --- a/daemon/id.go +++ b/daemon/id.go @@ -2,21 +2,25 @@ package daemon // import "github.com/docker/docker/daemon" import ( "os" + "path/filepath" "github.com/docker/docker/pkg/ioutils" "github.com/google/uuid" "github.com/pkg/errors" ) -// loadOrCreateID loads the engine's ID from idPath, or generates a new ID +const idFilename = "engine-id" + +// LoadOrCreateID loads the engine's ID from the given root, or generates a new ID // if it doesn't exist. It returns the ID, and any error that occurred when // saving the file. // // Note that this function expects the daemon's root directory to already have // been created with the right permissions and ownership (usually this would // be done by daemon.CreateDaemonRoot(). -func loadOrCreateID(idPath string) (string, error) { +func LoadOrCreateID(root string) (string, error) { var id string + idPath := filepath.Join(root, idFilename) idb, err := os.ReadFile(idPath) if os.IsNotExist(err) { id = uuid.New().String() diff --git a/integration/container/pidmode_linux_test.go b/integration/container/pidmode_linux_test.go index 6f83bf572d..aeb209a44b 100644 --- a/integration/container/pidmode_linux_test.go +++ b/integration/container/pidmode_linux_test.go @@ -40,9 +40,8 @@ func TestPIDModeHost(t *testing.T) { func TestPIDModeContainer(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") - defer setupTest(t)() + ctx := setupTest(t) apiClient := testEnv.APIClient() - ctx := context.Background() t.Run("non-existing container", func(t *testing.T) { _, err := container.CreateFromConfig(ctx, apiClient, container.NewTestConfig(container.WithPIDMode("container:nosuchcontainer"))) diff --git a/integration/image/remove_test.go b/integration/image/remove_test.go index 6401313427..59684a1752 100644 --- a/integration/image/remove_test.go +++ b/integration/image/remove_test.go @@ -63,8 +63,7 @@ func TestRemoveImageOrphaning(t *testing.T) { func TestRemoveByDigest(t *testing.T) { skip.If(t, !testEnv.UsingSnapshotter(), "RepoDigests doesn't include tags when using graphdrivers") - defer setupTest(t)() - ctx := context.Background() + ctx := setupTest(t) client := testEnv.APIClient() err := client.ImageTag(ctx, "busybox", "test-remove-by-digest:latest") diff --git a/libcontainerd/supervisor/remote_daemon.go b/libcontainerd/supervisor/remote_daemon.go index 98a8a3cb73..4ef1c2ac08 100644 --- a/libcontainerd/supervisor/remote_daemon.go +++ b/libcontainerd/supervisor/remote_daemon.go @@ -10,6 +10,7 @@ import ( "time" "github.com/containerd/containerd" + "github.com/containerd/containerd/defaults" "github.com/containerd/containerd/log" "github.com/containerd/containerd/services/server/config" "github.com/containerd/containerd/sys" @@ -19,6 +20,9 @@ import ( "github.com/pelletier/go-toml" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -295,7 +299,17 @@ func (r *remote) monitorDaemon(ctx context.Context) { continue } - client, err = containerd.New(r.GRPC.Address, containerd.WithTimeout(60*time.Second)) + client, err = containerd.New( + r.GRPC.Address, + containerd.WithTimeout(60*time.Second), + containerd.WithDialOpts([]grpc.DialOption{ + grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), + }), + ) if err != nil { r.logger.WithError(err).Error("failed connecting to containerd") delay = 100 * time.Millisecond diff --git a/vendor.mod b/vendor.mod index 62e979e088..d124ce1464 100644 --- a/vendor.mod +++ b/vendor.mod @@ -92,7 +92,12 @@ require ( github.com/vishvananda/netns v0.0.2 go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 go.opentelemetry.io/otel v1.4.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.4.1 // indirect + go.opentelemetry.io/otel/sdk v1.4.1 go.opentelemetry.io/otel/trace v1.4.1 golang.org/x/mod v0.10.0 golang.org/x/net v0.10.0 @@ -187,14 +192,9 @@ require ( go.etcd.io/etcd/server/v3 v3.5.6 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.29.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.4.1 // indirect go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect go.opentelemetry.io/otel/metric v0.27.0 // indirect - go.opentelemetry.io/otel/sdk v1.4.1 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect diff --git a/vendor/github.com/containerd/containerd/tracing/log.go b/vendor/github.com/containerd/containerd/tracing/log.go new file mode 100644 index 0000000000..6c6dd6d7e6 --- /dev/null +++ b/vendor/github.com/containerd/containerd/tracing/log.go @@ -0,0 +1,130 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tracing + +import ( + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// NewLogrusHook creates a new logrus hook +func NewLogrusHook() *LogrusHook { + return &LogrusHook{} +} + +// LogrusHook is a logrus hook which adds logrus events to active spans. +// If the span is not recording or the span context is invalid, the hook is a no-op. +type LogrusHook struct{} + +// Levels returns the logrus levels that this hook is interested in. +func (h *LogrusHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +// Fire is called when a log event occurs. +func (h *LogrusHook) Fire(entry *logrus.Entry) error { + span := trace.SpanFromContext(entry.Context) + if span == nil { + return nil + } + + if !span.SpanContext().IsValid() || !span.IsRecording() { + return nil + } + + span.AddEvent( + entry.Message, + trace.WithAttributes(logrusDataToAttrs(entry.Data)...), + trace.WithAttributes(attribute.String("level", entry.Level.String())), + trace.WithTimestamp(entry.Time), + ) + + return nil +} + +func logrusDataToAttrs(data logrus.Fields) []attribute.KeyValue { + attrs := make([]attribute.KeyValue, 0, len(data)) + for k, v := range data { + attrs = append(attrs, any(k, v)) + } + return attrs +} + +func any(k string, v interface{}) attribute.KeyValue { + if v == nil { + return attribute.String(k, "") + } + + switch typed := v.(type) { + case bool: + return attribute.Bool(k, typed) + case []bool: + return attribute.BoolSlice(k, typed) + case int: + return attribute.Int(k, typed) + case []int: + return attribute.IntSlice(k, typed) + case int8: + return attribute.Int(k, int(typed)) + case []int8: + ls := make([]int, 0, len(typed)) + for _, i := range typed { + ls = append(ls, int(i)) + } + return attribute.IntSlice(k, ls) + case int16: + return attribute.Int(k, int(typed)) + case []int16: + ls := make([]int, 0, len(typed)) + for _, i := range typed { + ls = append(ls, int(i)) + } + return attribute.IntSlice(k, ls) + case int32: + return attribute.Int64(k, int64(typed)) + case []int32: + ls := make([]int64, 0, len(typed)) + for _, i := range typed { + ls = append(ls, int64(i)) + } + return attribute.Int64Slice(k, ls) + case int64: + return attribute.Int64(k, typed) + case []int64: + return attribute.Int64Slice(k, typed) + case float64: + return attribute.Float64(k, typed) + case []float64: + return attribute.Float64Slice(k, typed) + case string: + return attribute.String(k, typed) + case []string: + return attribute.StringSlice(k, typed) + } + + if stringer, ok := v.(fmt.Stringer); ok { + return attribute.String(k, stringer.String()) + } + if b, err := json.Marshal(v); b != nil && err == nil { + return attribute.String(k, string(b)) + } + return attribute.String(k, fmt.Sprintf("%v", v)) +} diff --git a/vendor/github.com/containerd/containerd/tracing/tracing.go b/vendor/github.com/containerd/containerd/tracing/tracing.go new file mode 100644 index 0000000000..d3ecfb5f9b --- /dev/null +++ b/vendor/github.com/containerd/containerd/tracing/tracing.go @@ -0,0 +1,37 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +// StartSpan starts child span in a context. +func StartSpan(ctx context.Context, opName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + if parent := trace.SpanFromContext(ctx); parent != nil && parent.SpanContext().IsValid() { + return parent.TracerProvider().Tracer("").Start(ctx, opName, opts...) + } + return otel.Tracer("").Start(ctx, opName, opts...) +} + +// StopSpan ends the span specified +func StopSpan(span trace.Span) { + span.End() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4f0ce02e9e..6b8c493628 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -319,6 +319,7 @@ github.com/containerd/containerd/snapshots/overlay/overlayutils github.com/containerd/containerd/snapshots/proxy github.com/containerd/containerd/sys github.com/containerd/containerd/sys/reaper +github.com/containerd/containerd/tracing github.com/containerd/containerd/version # github.com/containerd/continuity v0.4.1 ## explicit; go 1.19