diff --git a/fe/http.go b/fe/http.go
index d204401c18b958340a1c272d263bf0a1f4305243..2d0854af33195dafd6300e298ad01da03ae17468 100644
--- a/fe/http.go
+++ b/fe/http.go
@@ -371,7 +371,12 @@ func (h *HttpRedirector) serveStatusPage(w http.ResponseWriter, r *http.Request)
 
 func withLocalhost(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if ip := net.ParseIP(r.RemoteAddr); !ip.IsLoopback() {
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+		if ip := net.ParseIP(host); !ip.IsLoopback() {
 			http.Error(w, "Unauthorized", http.StatusUnauthorized)
 			return
 		}