Add instrumentation

Starting with counters for authentication requests, split by service
and status.

cmd/auth-server/main.go

@@ -4,18 +4,21 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 
 	"git.autistici.org/id/auth/server"
 	"github.com/coreos/go-systemd/daemon"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
 	configPath              = flag.String("config", "/etc/auth-server/config.yml", "configuration `file`")
 	socketPath              = flag.String("socket", "/run/auth/socket", "`path` to the UNIX socket to listen on")
 	systemdSocketActivation = flag.Bool("systemd-socket", false, "use SystemD socket activation")
+	httpAddr                = flag.String("http-addr", "", "if not nil, bind an HTTP server to this `addr` for Prometheus metrics")
 )
 
 func usage() {
@@ -44,6 +47,14 @@ func main() {
 		log.Fatalf("configuration error: %v", err)
 	}
 
+	if *httpAddr != "" {
+		h := http.NewServeMux()
+		h.Handle("/metrics", promhttp.Handler())
+		go func() {
+			log.Fatal(http.ListenAndServe(*httpAddr, h))
+		}()
+	}
+
 	var sockSrv *server.SocketServer
 	if *systemdSocketActivation {
 		sockSrv, err = server.NewSystemdSocketServer(authSrv)

server/authserver.go

@@ -11,6 +11,7 @@ import (
 
 	scrypt "github.com/elithrar/simple-scrypt"
 	"github.com/pquerna/otp/totp"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/tstranex/u2f"
 	"gopkg.in/yaml.v2"
 
@@ -301,6 +302,29 @@ func LoadConfig(path string) (*Config, error) {
 	return &config, nil
 }
 
+// Instrumentation.
+var (
+	authRequestsCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "auth_requests",
+			Help: "Number of authentication requests.",
+		},
+		[]string{"service", "status"},
+	)
+	ratelimitCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "auth_requests_ratelimited",
+			Help: "Number of rate-limited authentication requests.",
+		},
+		[]string{"service"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(authRequestsCounter)
+	prometheus.MustRegister(ratelimitCounter)
+}
+
 // Server is the main authentication server object.
 type Server struct {
 	backends     []UserBackend
@@ -377,29 +401,43 @@ func (s *Server) Authenticate(ctx context.Context, req *auth.Request) *auth.Resp
 
 	user, ok := s.getUser(ctx, serviceConfig, req.Username)
 	if !ok {
-		// User is unknown to all backends.
+		// User is unknown to all backends. Do not proceed
+		// further, but log and increment stats counters.
 		log.Printf("unknown user %s", req.Username)
+		authRequestsCounter.With(prometheus.Labels{
+			"service": req.Service,
+			"status":  "unknown_user",
+		})
 		return newError()
 	}
 
 	// Apply rate limiting and blacklisting _before_ invoking the
 	// authentication handlers, as they may be CPU intensive.
 	if allowed := serviceConfig.checkRateLimits(user, req); !allowed {
+		ratelimitCounter.With(prometheus.Labels{
+			"service": req.Service,
+		}).Inc()
 		return newError()
 	}
 
 	resp, err := s.authenticateUser(req, serviceConfig, user)
 	if err != nil {
 		resp = newError()
-		log.Printf("auth: user=%s status=%s error=%s", req.Username, resp.Status.String(), err)
+		log.Printf("auth: user=%s service=%s status=%s error=%s", req.Username, req.Service, resp.Status.String(), err)
 	} else {
 		// Log the request and response.
-		log.Printf("auth: user=%s status=%s", req.Username, resp.Status.String())
+		log.Printf("auth: user=%s service=%s status=%s", req.Username, req.Service, resp.Status.String())
 	}
 
 	// Notify blacklists of the result.
 	serviceConfig.notifyBlacklists(user, req, resp)
 
+	// Increment stats counters.
+	authRequestsCounter.With(prometheus.Labels{
+		"service": req.Service,
+		"status":  resp.Status.String(),
+	}).Inc()
+
 	return resp
 }