builder-next: enable OTLP tracing for history records

This enables picking up OTLP tracing context for the gRPC
requests.

Also sets up the in-memory recorder that BuildKit History API
can use to store the traces associated with specific build
in a database after build completes.

This doesn't enable Jaeger tracing endpoints from env
but this can be easily enabled by adding another import if
maintainers want it.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

api/server/router/grpc/grpc.go

@@ -1,26 +1,52 @@
 package grpc // import "github.com/docker/docker/api/server/router/grpc"
 
 import (
+	"context"
+	"strings"
+
 	"github.com/docker/docker/api/server/router"
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/moby/buildkit/util/grpcerrors"
+	"github.com/moby/buildkit/util/tracing/detect"
+	"github.com/sirupsen/logrus"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/net/http2"
 	"google.golang.org/grpc"
 )
 
+func init() {
+	// enable in memory recording for grpc traces
+	detect.Recorder = detect.NewTraceRecorder()
+}
+
 type grpcRouter struct {
 	routes     []router.Route
 	grpcServer *grpc.Server
 	h2Server   *http2.Server
 }
 
+var propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
+
 // NewRouter initializes a new grpc http router
 func NewRouter(backends ...Backend) router.Router {
+	tp, err := detect.TracerProvider()
+	if err != nil {
+		logrus.WithError(err).Error("failed to detect trace provider")
+	}
+
+	opts := []grpc.ServerOption{grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor)}
+	if tp != nil {
+		streamTracer := otelgrpc.StreamServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators))
+		unary := grpc_middleware.ChainUnaryServer(unaryInterceptor(tp), grpcerrors.UnaryServerInterceptor)
+		stream := grpc_middleware.ChainStreamServer(streamTracer, grpcerrors.StreamServerInterceptor)
+		opts = []grpc.ServerOption{grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream)}
+	}
+
 	r := &grpcRouter{
-		h2Server: &http2.Server{},
-		grpcServer: grpc.NewServer(
-			grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor),
-			grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor),
-		),
+		h2Server:   &http2.Server{},
+		grpcServer: grpc.NewServer(opts...),
 	}
 	for _, b := range backends {
 		b.RegisterGRPC(r.grpcServer)
@@ -39,3 +65,17 @@ func (gr *grpcRouter) initRoutes() {
 		router.NewPostRoute("/grpc", gr.serveGRPC),
 	}
 }
+
+func unaryInterceptor(tp trace.TracerProvider) grpc.UnaryServerInterceptor {
+	withTrace := otelgrpc.UnaryServerInterceptor(otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators(propagators))
+
+	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+		// This method is used by the clients to send their traces to buildkit so they can be included
+		// in the daemon trace and stored in the build history record. This method can not be traced because
+		// it would cause an infinite loop.
+		if strings.HasSuffix(info.FullMethod, "opentelemetry.proto.collector.trace.v1.TraceService/Export") {
+			return handler(ctx, req)
+		}
+		return withTrace(ctx, req, info, handler)
+	}
+}

vendor.mod

@@ -46,6 +46,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/go-immutable-radix v1.3.1
 	github.com/hashicorp/go-memdb v1.3.2
 	github.com/hashicorp/go-multierror v1.1.1
@@ -87,6 +88,9 @@ require (
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	github.com/vishvananda/netns v0.0.2
 	go.etcd.io/bbolt v1.3.7
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0
+	go.opentelemetry.io/otel v1.4.1
+	go.opentelemetry.io/otel/trace v1.4.1
 	golang.org/x/mod v0.10.0
 	golang.org/x/net v0.8.0
 	golang.org/x/sync v0.1.0
@@ -142,7 +146,6 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -172,10 +175,8 @@ require (
 	go.etcd.io/etcd/raft/v3 v3.5.6 // indirect
 	go.etcd.io/etcd/server/v3 v3.5.6 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.29.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 // indirect
-	go.opentelemetry.io/otel v1.4.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.4.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.4.1 // indirect
@@ -183,7 +184,6 @@ require (
 	go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect
 	go.opentelemetry.io/otel/metric v0.27.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.4.1 // indirect
-	go.opentelemetry.io/otel/trace v1.4.1 // indirect
 	go.opentelemetry.io/proto/otlp v0.12.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect