diff --git a/clientutil/transport.go b/clientutil/transport.go
index 843a760b1e25eb1c21b2c8552b9c6ad107ea6d34..1cc6c6d34594869c8be446fd325c27d1ad2c3289 100644
--- a/clientutil/transport.go
+++ b/clientutil/transport.go
@@ -7,6 +7,8 @@ import (
"net/http"
"sync"
"time"
+
+ "git.autistici.org/ai3/go-common/tracing"
)
// The transportCache is just a cache of http transports, each
@@ -29,12 +31,12 @@ func newTransportCache(tlsConfig *tls.Config) *transportCache {
}
func (m *transportCache) newTransport(addr string) http.RoundTripper {
- return &http.Transport{
+ return tracing.WrapTransport(&http.Transport{
TLSClientConfig: m.tlsConfig,
DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
return netDialContext(ctx, network, addr)
},
- }
+ })
}
func (m *transportCache) getTransport(addr string) http.RoundTripper {
diff --git a/serverutil/http.go b/serverutil/http.go
index 09cc9bb39440f01b502cf375d9a3ec28777e4955..604ca98f54357b6d423f341f8e71cd589697555a 100644
--- a/serverutil/http.go
+++ b/serverutil/http.go
@@ -3,16 +3,18 @@ package serverutil
import (
"context"
"crypto/tls"
+ "fmt"
"io"
"log"
"net"
"net/http"
- "net/http/pprof"
+ _ "net/http/pprof"
"os"
"os/signal"
"syscall"
"time"
+ "git.autistici.org/ai3/go-common/tracing"
"github.com/coreos/go-systemd/daemon"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -77,6 +79,10 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
// the listener, otherwise it will handle graceful termination on
// SIGINT or SIGTERM and return nil.
func Serve(h http.Handler, config *ServerConfig, addr string) error {
+ // Wrap with tracing handler (exclude metrics and other
+ // debugging endpoints).
+ h = tracing.WrapHandler(h, guessEndpointName(addr))
+
// Create the HTTP server.
srv, err := config.buildHTTPServer(h)
if err != nil {
@@ -139,8 +145,10 @@ func defaultHandler(h http.Handler) http.Handler {
// Add an endpoint to serve Prometheus metrics.
root.Handle("/metrics", promhttp.Handler())
- // Add the net/http/pprof debug handlers.
- root.Handle("/debug/pprof/", pprof.Handler(""))
+ // Let the default net/http handler deal with /debug/
+ // URLs. Packages such as net/http/pprof register their
+ // handlers there in ways that aren't reproducible.
+ root.Handle("/debug/", http.DefaultServeMux)
// Forward everything else to the main handler, adding
// Prometheus instrumentation (requests to /metrics and
@@ -151,6 +159,18 @@ func defaultHandler(h http.Handler) http.Handler {
return root
}
+func guessEndpointName(addr string) string {
+ _, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ return addr
+ }
+ host, err := os.Hostname()
+ if err != nil {
+ return addr
+ }
+ return fmt.Sprintf("%s:%s", host, port)
+}
+
// HTTP-related metrics.
var (
// Since we instrument the root HTTP handler, we don't really
diff --git a/tracing/tracing.go b/tracing/tracing.go
new file mode 100644
index 0000000000000000000000000000000000000000..df6144b7dd054b4dfde6640d95351f872050275a
--- /dev/null
+++ b/tracing/tracing.go
@@ -0,0 +1,130 @@
+package tracing
+
+import (
+ "encoding/json"
+ "errors"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sync"
+
+ openzipkin "github.com/openzipkin/zipkin-go"
+ zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http"
+ "go.opencensus.io/exporter/zipkin"
+ "go.opencensus.io/plugin/ochttp"
+ "go.opencensus.io/trace"
+)
+
+var (
+ // Enabled reports whether tracing is globally enabled or not.
+ Enabled bool
+
+ // The active tracing configuration, if Enabled is true.
+ config tracingConfig
+
+ initOnce sync.Once
+)
+
+const globalTracingConfigPath = "/etc/tracing/client.conf"
+
+type tracingConfig struct {
+ ReportURL string `json:"report_url"`
+}
+
+// Read the global tracing configuration file. Its location is
+// hardcoded, but it can be overriden using the TRACING_CONFIG
+// environment variable.
+func readTracingConfig() error {
+ // Read and decode configuration.
+ cfgPath := globalTracingConfigPath
+ if s := os.Getenv("TRACING_CONFIG"); s != "" {
+ cfgPath = s
+ }
+ data, err := ioutil.ReadFile(cfgPath)
+ if err != nil {
+ return err
+ }
+
+ if err := json.Unmarshal(data, &config); err != nil {
+ log.Printf("warning: error in tracing configuration: %v, tracing disabled", err)
+ return err
+ }
+
+ if config.ReportURL == "" {
+ log.Printf("warning: tracing configuration contains no report_url, tracing disabled")
+ return errors.New("no report_url")
+ }
+
+ return nil
+}
+
+// Compute the service name for Zipkin: this is usually the program
+// name (without path), but it can be overriden by the TRACING_SERVICE
+// environment variable.
+func getServiceName() string {
+ if s := os.Getenv("TRACING_SERVICE"); s != "" {
+ return s
+ }
+ return filepath.Base(os.Args[0])
+}
+
+// Initialize tracing. Tracing will be enabled if the system-wide
+// tracing configuration file is present and valid. Explicitly set
+// TRACING_ENABLE=0 in the environment to disable tracing.
+//
+// We need to check the configuration as soon as possible, because
+// it's likely that client transports are created before HTTP servers,
+// and we need to wrap them with opencensus at creation time.
+func init() {
+ // Kill switch from environment.
+ if s := os.Getenv("TRACING_ENABLE"); s == "0" {
+ return
+ }
+
+ if err := readTracingConfig(); err != nil {
+ return
+ }
+
+ Enabled = true
+}
+
+func initTracing(endpointAddr string) {
+ initOnce.Do(func() {
+ localEndpoint, err := openzipkin.NewEndpoint(getServiceName(), endpointAddr)
+ if err != nil {
+ log.Printf("warning: error creating tracing endpoint: %v, tracing disabled", err)
+ return
+ }
+
+ reporter := zipkinHTTP.NewReporter(config.ReportURL)
+ ze := zipkin.NewExporter(reporter, localEndpoint)
+
+ trace.RegisterExporter(ze)
+ trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+
+ log.Printf("tracing enabled (report_url %s)", config.ReportURL)
+
+ Enabled = true
+ })
+}
+
+// WrapTransport optionally wraps a http.RoundTripper with OpenCensus
+// tracing functionality, if it is globally enabled.
+func WrapTransport(t http.RoundTripper) http.RoundTripper {
+ if Enabled {
+ t = &ochttp.Transport{Base: t}
+ }
+ return t
+}
+
+// WrapHandler wraps a http.Handler with OpenCensus tracing
+// functionality, if globally enabled.
+func WrapHandler(h http.Handler, endpointAddr string) http.Handler {
+ if Enabled {
+ initTracing(endpointAddr)
+ h = &ochttp.Handler{Handler: h}
+ }
+ return h
+}