Minor http server improvements

Fixes include:
* only notify systemd after the network socket has been opened
* add net/http/pprof debug handlers
* add /health endpoint for Prometheus blackbox prober

serverutil/http.go

@@ -3,8 +3,11 @@ package serverutil
 import (
 	"context"
 	"crypto/tls"
+	"io"
 	"log"
+	"net"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
@@ -23,7 +26,7 @@ type ServerConfig struct {
 	TrustedForwarders   []string         `yaml:"trusted_forwarders"`
 }
 
-func (config *ServerConfig) buildHTTPServer(h http.Handler, addr string) (*http.Server, error) {
+func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error) {
 	var tlsConfig *tls.Config
 	var err error
 	if config != nil {
@@ -58,8 +61,7 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler, addr string) (*http.
 	// can be generous with the timeouts to keep the number of
 	// reconnections low.
 	return &http.Server{
-		Addr:         addr,
-		Handler:      instrumentHandler(h),
+		Handler:      defaultHandler(h),
 		ReadTimeout:  30 * time.Second,
 		WriteTimeout: 30 * time.Second,
 		IdleTimeout:  600 * time.Second,
@@ -74,11 +76,21 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler, addr string) (*http.
 // the listener, otherwise it will handle graceful termination on
 // SIGINT or SIGTERM and return nil.
 func Serve(h http.Handler, config *ServerConfig, addr string) error {
-	srv, err := config.buildHTTPServer(h, addr)
+	// Create the HTTP server.
+	srv, err := config.buildHTTPServer(h)
 	if err != nil {
 		return err
 	}
 
+	// Create the net.Listener, then notify systemd that we are ready to serve.
+	l, err := net.Listen("tcp", addr)
+	if err != nil {
+		return err
+	}
+	if srv.TLSConfig != nil {
+		l = tls.NewListener(l, srv.TLSConfig)
+	}
+
 	// Install a signal handler for gentle process termination.
 	done := make(chan struct{})
 	sigCh := make(chan os.Signal, 1)
@@ -98,13 +110,10 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error {
 
 		close(done)
 	}()
+
 	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
 
-	if srv.TLSConfig != nil {
-		err = srv.ListenAndServeTLS("", "")
-	} else {
-		err = srv.ListenAndServe()
-	}
+	err = srv.Serve(l)
 	if err != http.ErrServerClosed {
 		return err
 	}
@@ -113,12 +122,27 @@ func Serve(h http.Handler, config *ServerConfig, addr string) error {
 	return nil
 }
 
-func instrumentHandler(h http.Handler) http.Handler {
+func defaultHandler(h http.Handler) http.Handler {
 	root := http.NewServeMux()
+
+	// Add an endpoint for HTTP health checking probes.
+	root.Handle("/health", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		io.WriteString(w, "OK")
+	}))
+
+	// Add an endpoint to serve Prometheus metrics.
 	root.Handle("/metrics", promhttp.Handler())
-	root.Handle("/", h)
-	return promhttp.InstrumentHandlerInFlight(inFlightRequests,
-		promhttp.InstrumentHandlerCounter(totalRequests, root))
+
+	// Add the net/http/pprof debug handlers.
+	root.Handle("/debug/pprof/", pprof.Handler(""))
+
+	// Forward everything else to the main handler, adding
+	// Prometheus instrumentation (requests to /metrics and
+	// /health are not included).
+	root.Handle("/", promhttp.InstrumentHandlerInFlight(inFlightRequests,
+		promhttp.InstrumentHandlerCounter(totalRequests, h)))
+
+	return root
 }
 
 // HTTP-related metrics.