Add some instrumentation to the statusManager

node/instrumentation.go

@@ -52,6 +52,20 @@ var (
 		},
 	)
 
+	// Status protocol (gossip) metrics.
+	gossipNumNodes = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "gossip_peer_count",
+			Help: "Number of peers seen by the gossip protocol.",
+		},
+	)
+	gossipOldestTS = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "gossip_oldest_ts",
+			Help: "Timestamp of the oldest update from any active peer.",
+		},
+	)
+
 	// Descriptors for the nodeCollector below.
 	numListenersDesc = prometheus.NewDesc(
 		"status_num_listeners",

node/status.go

@@ -149,6 +149,24 @@ func (m *statusManager) mergeRemoteStatuses(remote []*pb.Status) {
 	m.statuses = s
 }
 
+// Export the number of known nodes, and the timestamp of the oldest
+// update seen, to monitoring, so we can have a rough idea of when the
+// gossip protocol isn't working.
+func (m *statusManager) updateMetrics() {
+	m.mx.Lock()
+	var oldest uint64
+	for _, s := range m.statuses {
+		t := s.Timestamp
+		if oldest == 0 || t < oldest {
+			oldest = t
+		}
+	}
+
+	gossipNumNodes.Set(float64(len(m.statuses)))
+	gossipOldestTS.Set(float64(oldest))
+	m.mx.Unlock()
+}
+
 func (m *statusManager) tick(ctx context.Context) {
 	pctx, cancel := context.WithTimeout(ctx, gossipTimeout)
 	defer cancel()
@@ -157,6 +175,8 @@ func (m *statusManager) tick(ctx context.Context) {
 	if err != nil && err != context.Canceled {
 		log.Printf("status: gossip error: %v", err)
 	}
+
+	m.updateMetrics()
 }
 
 func (m *statusManager) Exchange(ctx context.Context, req *pb.ExchangeRequest) (*pb.ExchangeResponse, error) {