Add specific instrumentation for Keystore operations

server/instrumentation.go

+package server
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	totalKeysInMemory = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "keystored_keys_total",
+		Help: "Total number of unlocked keys in-memory.",
+	})
+	requestsCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "keystored_requests_total",
+		Help: "Counter of requests by method and status.",
+	}, []string{"method", "status"})
+	decryptedKeysCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "keystored_decrypted_keys_total",
+		Help: "Counter of decrypted keys.",
+	})
+	unlockedKeysServedCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "keystored_unlocked_keys_served_total",
+		Help: "Counter of unlocked keys served.",
+	})
+)
+
+func (s *KeyStore) updateKeyspaceSize() {
+	totalKeysInMemory.Set(float64(len(s.userKeys)))
+}

server/keystore.go

@@ -127,6 +127,7 @@ func (s *KeyStore) expire(t time.Time) {
 			delete(s.userKeys, u)
 		}
 	}
+	s.updateKeyspaceSize()
 	s.mx.Unlock()
 }
 
@@ -173,6 +174,7 @@ func (s *KeyStore) Open(ctx context.Context, username, password string, ttlSecon
 		pkey:   pem,
 		expiry: time.Now().Add(time.Duration(ttlSeconds) * time.Second),
 	}
+	s.updateKeyspaceSize()
 	s.mx.Unlock()
 	return nil
 }
@@ -210,6 +212,7 @@ func (s *KeyStore) Close(username string) bool {
 	if ok {
 		wipeBytes(k.pkey)
 		delete(s.userKeys, username)
+		s.updateKeyspaceSize()
 	}
 	return ok
 }

server/server.go

@@ -27,11 +27,14 @@ func (s *keyStoreServer) handleOpen(w http.ResponseWriter, r *http.Request) {
 	} else if err != nil {
 		log.Printf("Open(%s): error: %v", req.Username, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
+		requestsCounter.WithLabelValues("Open", "error").Inc()
 		return
 	} else {
 		log.Printf("Open(%s): decrypted key, ttl=%d", req.Username, req.TTL)
+		decryptedKeysCounter.Inc()
 	}
 
+	requestsCounter.WithLabelValues("Open", "ok").Inc()
 	serverutil.EncodeJSONResponse(w, &emptyResponse)
 }
 
@@ -54,13 +57,16 @@ func (s *keyStoreServer) handleGet(w http.ResponseWriter, r *http.Request) {
 			log.Printf("Get(%s): error: %v", req.Username, err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
+		requestsCounter.WithLabelValues("Get", "error").Inc()
 		return
 	} else {
 		resp.HasKey = true
 		resp.Key = key
 		log.Printf("Get(%s): fetched key", req.Username)
+		unlockedKeysServedCounter.Inc()
 	}
 
+	requestsCounter.WithLabelValues("Get", "ok").Inc()
 	serverutil.EncodeJSONResponse(w, &resp)
 }
 
@@ -74,6 +80,7 @@ func (s *keyStoreServer) handleClose(w http.ResponseWriter, r *http.Request) {
 		log.Printf("Close(%s): discarded key", req.Username)
 	}
 
+	requestsCounter.WithLabelValues("Close", "ok").Inc()
 	serverutil.EncodeJSONResponse(w, &emptyResponse)
 }