Add PSI metrics

cpu_v2.go

@@ -6,12 +6,29 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-var usecs float64 = 1000000
+var (
+	usecs float64 = 1000000
+
+	cpuV2PressureStalledDesc = prometheus.NewDesc(
+		"cgroup_cpu_pressure_stalled_seconds_total",
+		"PSI stalled CPU seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
+	cpuV2PressureWaitingDesc = prometheus.NewDesc(
+		"cgroup_cpu_pressure_waiting_seconds_total",
+		"PSI waiting CPU seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
+)
 
 type cpuV2Parser struct{}
 
 func (p *cpuV2Parser) describe(ch chan<- *prometheus.Desc) {
 	ch <- cpuV1Desc
+	ch <- cpuV2PressureStalledDesc
+	ch <- cpuV2PressureWaitingDesc
 }
 
 func (p *cpuV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Metric) {
@@ -33,4 +50,20 @@ func (p *cpuV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Metri
 		float64(usage["system_usec"])/usecs,
 		"system", slice, unit,
 	)
+
+	waiting, stalled, err := parsePressureFile(filepath.Join(cgroupsRootPath, path, "cpu.pressure"))
+	if err == nil {
+		ch <- prometheus.MustNewConstMetric(
+			cpuV2PressureWaitingDesc,
+			prometheus.CounterValue,
+			float64(waiting),
+			slice, unit,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			cpuV2PressureStalledDesc,
+			prometheus.CounterValue,
+			float64(stalled),
+			slice, unit,
+		)
+	}
 }

io_v2.go

@@ -24,6 +24,18 @@ var (
 		[]string{"mode", "slice", "service"},
 		nil,
 	)
+	ioV2PressureStalledDesc = prometheus.NewDesc(
+		"cgroup_blkio_pressure_stalled_seconds_total",
+		"PSI stalled I/O seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
+	ioV2PressureWaitingDesc = prometheus.NewDesc(
+		"cgroup_blkio_pressure_waiting_seconds_total",
+		"PSI waiting I/O seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
 )
 
 type blkioV2Parser struct{}
@@ -44,17 +56,23 @@ func parseDevice(token []byte) (int, int, error) {
 	return maj, min, nil
 }
 
-func parseKVPairs(tokens [][]byte, out map[string]int64) {
-	for _, token := range tokens {
+func parseKVPair(token []byte) (string, int64, error) {
 	kvp := bytes.SplitN(token, []byte("="), 2)
 	if len(kvp) != 2 {
-			continue
+		return "", 0, errors.New("not an assignment")
 	}
 	value, err := strconv.ParseInt(string(kvp[1]), 10, 64)
 	if err != nil {
-			continue
+		return "", 0, err
+	}
+	return string(kvp[0]), value, nil
+}
+
+func parseKVPairs(tokens [][]byte, out map[string]int64) {
+	for _, token := range tokens {
+		if key, value, err := parseKVPair(token); err == nil {
+			out[key] += value
 		}
-		out[string(kvp[0])] += value
 	}
 }
 
@@ -65,9 +83,8 @@ func parseIOV2File(path string) (map[string]int64, error) {
 	}
 	defer f.Close()
 
-	// Sum I/O counters across devices, but only for mounted
-	// devices, so we do not count I/O operations twice with
-	// LVM/MD.
+	// Sum I/O counters across devices, but only for mounted devices,
+	// so we do not count I/O operations twice with LVM/MD.
 	result := make(map[string]int64)
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
@@ -92,6 +109,8 @@ func parseIOV2File(path string) (map[string]int64, error) {
 func (p *blkioV2Parser) describe(ch chan<- *prometheus.Desc) {
 	ch <- ioV2BytesDesc
 	ch <- ioV2OpsDesc
+	ch <- ioV2PressureStalledDesc
+	ch <- ioV2PressureWaitingDesc
 }
 
 func (p *blkioV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Metric) {
@@ -124,4 +143,20 @@ func (p *blkioV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Met
 		float64(counters["rios"]),
 		"read", slice, unit,
 	)
+
+	waiting, stalled, err := parsePressureFile(filepath.Join(cgroupsRootPath, path, "io.pressure"))
+	if err == nil {
+		ch <- prometheus.MustNewConstMetric(
+			ioV2PressureWaitingDesc,
+			prometheus.CounterValue,
+			float64(waiting),
+			slice, unit,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			ioV2PressureStalledDesc,
+			prometheus.CounterValue,
+			float64(stalled),
+			slice, unit,
+		)
+	}
 }

mem_v2.go

@@ -6,10 +6,27 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+var (
+	memV2PressureStalledDesc = prometheus.NewDesc(
+		"cgroup_mem_pressure_stalled_seconds_total",
+		"PSI stalled memory seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
+	memV2PressureWaitingDesc = prometheus.NewDesc(
+		"cgroup_mem_pressure_waiting_seconds_total",
+		"PSI waiting memory seconds.",
+		[]string{"slice", "service"},
+		nil,
+	)
+)
+
 type memoryV2Parser struct{}
 
 func (p *memoryV2Parser) describe(ch chan<- *prometheus.Desc) {
 	ch <- memV1Desc
+	ch <- memV2PressureStalledDesc
+	ch <- memV2PressureWaitingDesc
 }
 
 func (p *memoryV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Metric) {
@@ -25,4 +42,20 @@ func (p *memoryV2Parser) parse(path, slice, unit string, ch chan<- prometheus.Me
 		float64(rss),
 		slice, unit,
 	)
+
+	waiting, stalled, err := parsePressureFile(filepath.Join(cgroupsRootPath, path, "memory.pressure"))
+	if err == nil {
+		ch <- prometheus.MustNewConstMetric(
+			memV2PressureWaitingDesc,
+			prometheus.CounterValue,
+			float64(waiting),
+			slice, unit,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			memV2PressureStalledDesc,
+			prometheus.CounterValue,
+			float64(stalled),
+			slice, unit,
+		)
+	}
 }

util.go

@@ -17,6 +17,8 @@ var (
 )
 
 func init() {
+	// Cgroups v1 counters are expressed in 'ticks', so we need to figure
+	// out the system's HZ value to convert them to seconds.
 	userHZ = 100
 	if clktck, err := sysconf.Sysconf(sysconf.SC_CLK_TCK); err == nil {
 		userHZ = float64(clktck)
@@ -27,6 +29,8 @@ func cgroupV1StatPath(cgroupPath, collector, path string) string {
 	return filepath.Join("/sys/fs/cgroup", collector, cgroupPath, path)
 }
 
+// Parse a generic proc-style 'map' file, with space-separated "key value"
+// assignments, one per line.
 func parseMapFile(path string) (map[string]int64, error) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -51,6 +55,7 @@ func parseMapFile(path string) (map[string]int64, error) {
 	return result, scanner.Err()
 }
 
+// Parse a file containing a single integer value.
 func parseSingleValueFile(path string) (int64, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
@@ -62,6 +67,37 @@ func parseSingleValueFile(path string) (int64, error) {
 	return strconv.ParseInt(string(data), 10, 64)
 }
 
+// Parse a PSI /proc file and return the "some" (waiting), "full" (stalled)
+// counters.
+func parsePressureFile(path string) (int64, int64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	defer f.Close()
+
+	var waiting, stalled int64
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		parts := bytes.Split(line, []byte(" "))
+		if len(parts) != 5 {
+			continue
+		}
+		_, value, err := parseKVPair(parts[4])
+		if err != nil {
+			continue
+		}
+		switch {
+		case bytes.Equal(parts[0], []byte("some")):
+			waiting = value
+		case bytes.Equal(parts[0], []byte("full")):
+			stalled = value
+		}
+	}
+	return waiting, stalled, scanner.Err()
+}
+
 func splitServiceName(path string) (string, string) {
 	slice, name := filepath.Split(path)
 	slice = strings.Trim(slice, "/")