Generate OpenTracing for Endpoints

[zaquestion]

Generate OpenTracing for Endpoints

[matthewhartstonge]

Heyo, didn't see this here before - I have this working in our fork as a generated middleware, would it be helpful to see how I've implemented it?

Currently I've got it generating extra files so as not to mash anything, and will be overwritten each time truss reruns. I had to pull in and customise a couple of funcs from go-kit v0.6.0, (I think they are in v0.5.0 were just renamed) as the logger in go-kit doesn't implement the stdlib logger interface.

There needed to be a couple of additions to the http/grpc transports as well to support incoming/outgoing transformation to OpenTracing standards i.e. creating HTTP Headers with the trace id.

Here's the bulk of the middlewares template:

package templates

const MiddlewaresTracing = `
// middlewares_tracing.go provides the ability to understanding request flows,
// identifying hot spots, and diagnose errors.
package handlers

import (
	// Standard Library Imports
	"context"
	"fmt"

	// External Imports
	"github.com/opentracing/opentracing-go"
	otext "github.com/opentracing/opentracing-go/ext"
	otlog "github.com/opentracing/opentracing-go/log"

	// Internal Imports
	pb "{{.PBImportPath -}}"
)

func TracingMiddleware(otTracer opentracing.Tracer) {{.Service.Name}}ServiceMiddleware {
	return func(next pb.{{.Service.Name}}Server) pb.{{.Service.Name}}Server {
		return tracingMiddleware{
			tracer: otTracer,
			next:   next,
		}
	}
}

// tracingMiddleware implements pb.{{.Service.Name}}Server in order to
// easily trace requests throughout a distributed system.
type tracingMiddleware struct {
	tracer opentracing.Tracer
	next   pb.{{.Service.Name}}Server
}

{{with $te := . }}
	{{range $i := $te.Service.Methods}}
		// {{$i.Name}} implements pb.{{$te.Service.Name}}Service in order to
		// provide distributed tracing visibility.
		func (tm tracingMiddleware) {{$i.Name}}(ctx context.Context, req *pb.{{GoName $i.RequestType.Name}}) (res *pb.{{GoName $i.ResponseType.Name}}, err error){
			serverSpan, tracedCtx := spanCreator(ctx, tm.tracer, "{{$i.Name}}")
			defer serverSpan.Finish()
			defer spanRecoverer(tracedCtx, serverSpan, "{{$i.Name}}")
		
			res, err = tm.next.{{$i.Name}}(tracedCtx, req)
			spanLogger(tracedCtx, serverSpan, res, err)
			return
		}
	{{end}}
{{- end}}

// spanCreator creates a new trace span throughout the distributed system.
func spanCreator(ctx context.Context, otTracer opentracing.Tracer, methodName string) (opentracing.Span, context.Context) {
	serverSpan := opentracing.SpanFromContext(ctx)
	if serverSpan == nil {
		// All we can do is create a new root span.
		serverSpan = otTracer.StartSpan(methodName)
	} else {
		serverSpan.SetOperationName(methodName)
	}
	otext.SpanKindRPCServer.Set(serverSpan)
	ctx = opentracing.ContextWithSpan(ctx, serverSpan)
	return serverSpan, ctx
}

// httpStatusCoder enables implementers to check for HTTP status codes
type httpStatusCoder interface {
	GetStatusCode() pb.StatusCode
}

// spanLogger reports the HTTP status code and any errors if found.
func spanLogger(ctx context.Context, span opentracing.Span, res httpStatusCoder, err error) {
	if err == nil {
		// simply log the returned status code
		span.SetTag(string(otext.HTTPStatusCode), res.GetStatusCode())
		return
	}

	// Deal with the nasty error...
	span.SetTag(string(otext.Error), true)
	span.LogFields(
		otlog.Error(err),
		otlog.String("tid", tid),
		otlog.String("sub", sub),
	)
	if res != nil {
		span.SetTag(string(otext.HTTPStatusCode), res.GetStatusCode())
	}
}

// spanRecoverer tells us information about the tenant, the user and the method
// they were accessing that happened to cause a blow out.
// Currently, there is not way to
func spanRecoverer(ctx context.Context, span opentracing.Span, method string) {
	if r := recover(); r != nil {
		// we've had a panic, record via opentracing
		span.SetTag(string(otext.Error), true)
		span.LogFields(
			otlog.String("method", method),
			otlog.String("error", fmt.Sprintf("%v", r)),
		)
	}
}
`

[zaquestion]

@matthewhartstonge This is fantastic, anything else to help ease implementation on our side would he much appreciated.