Add support for opencensus tracing

Trace requests on client and server HTTP traffic, using a system-wide
configuration file to control reporting.

clientutil/transport.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 	"sync"
 	"time"
+
+	"git.autistici.org/ai3/go-common/tracing"
 )
 
 // The transportCache is just a cache of http transports, each
@@ -29,12 +31,12 @@ func newTransportCache(tlsConfig *tls.Config) *transportCache {
 }
 
 func (m *transportCache) newTransport(addr string) http.RoundTripper {
-	return &http.Transport{
+	return tracing.WrapTransport(&http.Transport{
 		TLSClientConfig: m.tlsConfig,
 		DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
 			return netDialContext(ctx, network, addr)
 		},
-	}
+	})
 }
 
 func (m *transportCache) getTransport(addr string) http.RoundTripper {

serverutil/http.go

@@ -3,16 +3,18 @@ package serverutil
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"io"
 	"log"
 	"net"
 	"net/http"
-	"net/http/pprof"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"git.autistici.org/ai3/go-common/tracing"
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -77,6 +79,10 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
 // the listener, otherwise it will handle graceful termination on
 // SIGINT or SIGTERM and return nil.
 func Serve(h http.Handler, config *ServerConfig, addr string) error {
+	// Wrap with tracing handler (exclude metrics and other
+	// debugging endpoints).
+	h = tracing.WrapHandler(h, guessEndpointName(addr))
+
 	// Create the HTTP server.
 	srv, err := config.buildHTTPServer(h)
 	if err != nil {
@@ -139,8 +145,10 @@ func defaultHandler(h http.Handler) http.Handler {
 	// Add an endpoint to serve Prometheus metrics.
 	root.Handle("/metrics", promhttp.Handler())
 
-	// Add the net/http/pprof debug handlers.
-	root.Handle("/debug/pprof/", pprof.Handler(""))
+	// Let the default net/http handler deal with /debug/
+	// URLs. Packages such as net/http/pprof register their
+	// handlers there in ways that aren't reproducible.
+	root.Handle("/debug/", http.DefaultServeMux)
 
 	// Forward everything else to the main handler, adding
 	// Prometheus instrumentation (requests to /metrics and
@@ -151,6 +159,18 @@ func defaultHandler(h http.Handler) http.Handler {
 	return root
 }
 
+func guessEndpointName(addr string) string {
+	_, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return addr
+	}
+	host, err := os.Hostname()
+	if err != nil {
+		return addr
+	}
+	return fmt.Sprintf("%s:%s", host, port)
+}
+
 // HTTP-related metrics.
 var (
 	// Since we instrument the root HTTP handler, we don't really

tracing/tracing.go

+package tracing
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+
+	openzipkin "github.com/openzipkin/zipkin-go"
+	zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http"
+	"go.opencensus.io/exporter/zipkin"
+	"go.opencensus.io/plugin/ochttp"
+	"go.opencensus.io/trace"
+)
+
+var (
+	// Enabled reports whether tracing is globally enabled or not.
+	Enabled bool
+
+	// The active tracing configuration, if Enabled is true.
+	config tracingConfig
+
+	initOnce sync.Once
+)
+
+const globalTracingConfigPath = "/etc/tracing/client.conf"
+
+type tracingConfig struct {
+	ReportURL string `json:"report_url"`
+}
+
+// Read the global tracing configuration file. Its location is
+// hardcoded, but it can be overriden using the TRACING_CONFIG
+// environment variable.
+func readTracingConfig() error {
+	// Read and decode configuration.
+	cfgPath := globalTracingConfigPath
+	if s := os.Getenv("TRACING_CONFIG"); s != "" {
+		cfgPath = s
+	}
+	data, err := ioutil.ReadFile(cfgPath)
+	if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(data, &config); err != nil {
+		log.Printf("warning: error in tracing configuration: %v, tracing disabled", err)
+		return err
+	}
+
+	if config.ReportURL == "" {
+		log.Printf("warning: tracing configuration contains no report_url, tracing disabled")
+		return errors.New("no report_url")
+	}
+
+	return nil
+}
+
+// Compute the service name for Zipkin: this is usually the program
+// name (without path), but it can be overriden by the TRACING_SERVICE
+// environment variable.
+func getServiceName() string {
+	if s := os.Getenv("TRACING_SERVICE"); s != "" {
+		return s
+	}
+	return filepath.Base(os.Args[0])
+}
+
+// Initialize tracing. Tracing will be enabled if the system-wide
+// tracing configuration file is present and valid. Explicitly set
+// TRACING_ENABLE=0 in the environment to disable tracing.
+//
+// We need to check the configuration as soon as possible, because
+// it's likely that client transports are created before HTTP servers,
+// and we need to wrap them with opencensus at creation time.
+func init() {
+	// Kill switch from environment.
+	if s := os.Getenv("TRACING_ENABLE"); s == "0" {
+		return
+	}
+
+	if err := readTracingConfig(); err != nil {
+		return
+	}
+
+	Enabled = true
+}
+
+func initTracing(endpointAddr string) {
+	initOnce.Do(func() {
+		localEndpoint, err := openzipkin.NewEndpoint(getServiceName(), endpointAddr)
+		if err != nil {
+			log.Printf("warning: error creating tracing endpoint: %v, tracing disabled", err)
+			return
+		}
+
+		reporter := zipkinHTTP.NewReporter(config.ReportURL)
+		ze := zipkin.NewExporter(reporter, localEndpoint)
+
+		trace.RegisterExporter(ze)
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+
+		log.Printf("tracing enabled (report_url %s)", config.ReportURL)
+
+		Enabled = true
+	})
+}
+
+// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
+// tracing functionality, if it is globally enabled.
+func WrapTransport(t http.RoundTripper) http.RoundTripper {
+	if Enabled {
+		t = &ochttp.Transport{Base: t}
+	}
+	return t
+}
+
+// WrapHandler wraps a http.Handler with OpenCensus tracing
+// functionality, if globally enabled.
+func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
+	if Enabled {
+		initTracing(endpointAddr)
+		h = &ochttp.Handler{Handler: h}
+	}
+	return h
+}