diff --git a/serverutil/http.go b/serverutil/http.go
index 641c0b9bcae0001b66322e04b28c9da981617424..afaadf44f355adc0e342b3c761b4ce888a0ba57a 100644
--- a/serverutil/http.go
+++ b/serverutil/http.go
@@ -60,20 +60,19 @@ func (config *ServerConfig) buildHTTPServer(h http.Handler) (*http.Server, error
 		if config.MaxInflightRequests > 0 {
 			h = newLoadSheddingWrapper(config.MaxInflightRequests, h)
 		}
-	}
 
-	// Wrap the handler with a TimeoutHandler if 'request_timeout'
-	// is set.
-	h = addDefaultHandlers(h)
-	if config.RequestTimeoutSecs > 0 {
-		h = http.TimeoutHandler(h, time.Duration(config.RequestTimeoutSecs)*time.Second, "")
+		// Wrap the handler with a TimeoutHandler if 'request_timeout'
+		// is set.
+		if config.RequestTimeoutSecs > 0 {
+			h = http.TimeoutHandler(h, time.Duration(config.RequestTimeoutSecs)*time.Second, "")
+		}
 	}
 
 	// These are not meant to be external-facing servers, so we
 	// can be generous with the timeouts to keep the number of
 	// reconnections low.
 	return &http.Server{
-		Handler:      h,
+		Handler:      addDefaultHandlers(h),
 		ReadTimeout:  30 * time.Second,
 		WriteTimeout: 30 * time.Second,
 		IdleTimeout:  600 * time.Second,