Add a collector for /proc/key-users

This collects usage information (per-user) on the Linux kernel keyring
See: https://man7.org/linux/man-pages/man7/keyrings.7.html

It is disabled by default because cardinality is one-per user.

Signed-off-by: Robert Zimmerman <rzimmerman@cloudflare.com>

README.md

@@ -190,6 +190,7 @@ drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which ex
 drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux
 ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux
 interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD
+key_users | Linux kernel keyring utilization by user. | Linux
 ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux
 lnstat | Exposes stats from `/proc/net/stat/`. | Linux
 logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux

collector/fixtures/e2e-64k-page-output.txt

@@ -1291,6 +1291,55 @@ node_ipvs_outgoing_bytes_total 0
 # HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
 # TYPE node_ipvs_outgoing_packets_total counter
 node_ipvs_outgoing_packets_total 0
+# HELP node_keys_maxbytes Key utilization field maxbytes.
+# TYPE node_keys_maxbytes gauge
+node_keys_maxbytes{uid="0"} 2.5e+07
+node_keys_maxbytes{uid="1000"} 20000
+node_keys_maxbytes{uid="101"} 20000
+node_keys_maxbytes{uid="102"} 20000
+node_keys_maxbytes{uid="104"} 20000
+# HELP node_keys_maxkeys Key utilization field maxkeys.
+# TYPE node_keys_maxkeys gauge
+node_keys_maxkeys{uid="0"} 1e+06
+node_keys_maxkeys{uid="1000"} 200
+node_keys_maxkeys{uid="101"} 200
+node_keys_maxkeys{uid="102"} 200
+node_keys_maxkeys{uid="104"} 200
+# HELP node_keys_nikeys Key utilization field nikeys.
+# TYPE node_keys_nikeys gauge
+node_keys_nikeys{uid="0"} 50
+node_keys_nikeys{uid="1000"} 12
+node_keys_nikeys{uid="101"} 1
+node_keys_nikeys{uid="102"} 1
+node_keys_nikeys{uid="104"} 1
+# HELP node_keys_nkeys Key utilization field nkeys.
+# TYPE node_keys_nkeys gauge
+node_keys_nkeys{uid="0"} 50
+node_keys_nkeys{uid="1000"} 11
+node_keys_nkeys{uid="101"} 1
+node_keys_nkeys{uid="102"} 1
+node_keys_nkeys{uid="104"} 1
+# HELP node_keys_qnbytes Key utilization field qnbytes.
+# TYPE node_keys_qnbytes gauge
+node_keys_qnbytes{uid="0"} 883
+node_keys_qnbytes{uid="1000"} 250
+node_keys_qnbytes{uid="101"} 9
+node_keys_qnbytes{uid="102"} 9
+node_keys_qnbytes{uid="104"} 9
+# HELP node_keys_qnkeys Key utilization field qnkeys.
+# TYPE node_keys_qnkeys gauge
+node_keys_qnkeys{uid="0"} 44
+node_keys_qnkeys{uid="1000"} 13
+node_keys_qnkeys{uid="101"} 1
+node_keys_qnkeys{uid="102"} 1
+node_keys_qnkeys{uid="104"} 1
+# HELP node_keys_usage Key utilization field usage.
+# TYPE node_keys_usage gauge
+node_keys_usage{uid="0"} 51
+node_keys_usage{uid="1000"} 11
+node_keys_usage{uid="101"} 1
+node_keys_usage{uid="102"} 1
+node_keys_usage{uid="104"} 1
 # HELP node_ksmd_full_scans_total ksmd 'full_scans' file.
 # TYPE node_ksmd_full_scans_total counter
 node_ksmd_full_scans_total 323
@@ -2907,6 +2956,7 @@ node_scrape_collector_success{collector="hwmon"} 1
 node_scrape_collector_success{collector="infiniband"} 1
 node_scrape_collector_success{collector="interrupts"} 1
 node_scrape_collector_success{collector="ipvs"} 1
+node_scrape_collector_success{collector="key_users"} 1
 node_scrape_collector_success{collector="ksmd"} 1
 node_scrape_collector_success{collector="lnstat"} 1
 node_scrape_collector_success{collector="loadavg"} 1

collector/fixtures/e2e-output.txt

@@ -1313,6 +1313,55 @@ node_ipvs_outgoing_bytes_total 0
 # HELP node_ipvs_outgoing_packets_total The total number of outgoing packets.
 # TYPE node_ipvs_outgoing_packets_total counter
 node_ipvs_outgoing_packets_total 0
+# HELP node_keys_maxbytes Key utilization field maxbytes.
+# TYPE node_keys_maxbytes gauge
+node_keys_maxbytes{uid="0"} 2.5e+07
+node_keys_maxbytes{uid="1000"} 20000
+node_keys_maxbytes{uid="101"} 20000
+node_keys_maxbytes{uid="102"} 20000
+node_keys_maxbytes{uid="104"} 20000
+# HELP node_keys_maxkeys Key utilization field maxkeys.
+# TYPE node_keys_maxkeys gauge
+node_keys_maxkeys{uid="0"} 1e+06
+node_keys_maxkeys{uid="1000"} 200
+node_keys_maxkeys{uid="101"} 200
+node_keys_maxkeys{uid="102"} 200
+node_keys_maxkeys{uid="104"} 200
+# HELP node_keys_nikeys Key utilization field nikeys.
+# TYPE node_keys_nikeys gauge
+node_keys_nikeys{uid="0"} 50
+node_keys_nikeys{uid="1000"} 12
+node_keys_nikeys{uid="101"} 1
+node_keys_nikeys{uid="102"} 1
+node_keys_nikeys{uid="104"} 1
+# HELP node_keys_nkeys Key utilization field nkeys.
+# TYPE node_keys_nkeys gauge
+node_keys_nkeys{uid="0"} 50
+node_keys_nkeys{uid="1000"} 11
+node_keys_nkeys{uid="101"} 1
+node_keys_nkeys{uid="102"} 1
+node_keys_nkeys{uid="104"} 1
+# HELP node_keys_qnbytes Key utilization field qnbytes.
+# TYPE node_keys_qnbytes gauge
+node_keys_qnbytes{uid="0"} 883
+node_keys_qnbytes{uid="1000"} 250
+node_keys_qnbytes{uid="101"} 9
+node_keys_qnbytes{uid="102"} 9
+node_keys_qnbytes{uid="104"} 9
+# HELP node_keys_qnkeys Key utilization field qnkeys.
+# TYPE node_keys_qnkeys gauge
+node_keys_qnkeys{uid="0"} 44
+node_keys_qnkeys{uid="1000"} 13
+node_keys_qnkeys{uid="101"} 1
+node_keys_qnkeys{uid="102"} 1
+node_keys_qnkeys{uid="104"} 1
+# HELP node_keys_usage Key utilization field usage.
+# TYPE node_keys_usage gauge
+node_keys_usage{uid="0"} 51
+node_keys_usage{uid="1000"} 11
+node_keys_usage{uid="101"} 1
+node_keys_usage{uid="102"} 1
+node_keys_usage{uid="104"} 1
 # HELP node_ksmd_full_scans_total ksmd 'full_scans' file.
 # TYPE node_ksmd_full_scans_total counter
 node_ksmd_full_scans_total 323
@@ -2929,6 +2978,7 @@ node_scrape_collector_success{collector="hwmon"} 1
 node_scrape_collector_success{collector="infiniband"} 1
 node_scrape_collector_success{collector="interrupts"} 1
 node_scrape_collector_success{collector="ipvs"} 1
+node_scrape_collector_success{collector="key_users"} 1
 node_scrape_collector_success{collector="ksmd"} 1
 node_scrape_collector_success{collector="lnstat"} 1
 node_scrape_collector_success{collector="loadavg"} 1

collector/fixtures/proc/key-users

@@ -0,0 +1,5 @@
+    0:    51 50/50 44/1000000 883/25000000
+  101:     1 1/1 1/200 9/20000
+  102:     1 1/1 1/200 9/20000
+  104:     1 1/1 1/200 9/20000
+ 1000:    11 11/12 13/200 250/20000

collector/key_users_linux.go

@@ -0,0 +1,111 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux && !nokey_users
+// +build linux,!nokey_users
+
+package collector
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strconv"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	keyUsersSubsystem = "keys"
+)
+
+var (
+	// See https://man7.org/linux/man-pages/man7/keyrings.7.html
+	keyUsersLineRegex = regexp.MustCompile(`^\s*(?P<uid>\d+):\s+(?P<usage>\d+)\s+(?P<nkeys>\d+)/(?P<nikeys>\d+)\s+(?P<qnkeys>\d+)/(?P<maxkeys>\d+)\s+(?P<qnbytes>\d+)/(?P<maxbytes>\d+)$`)
+)
+
+type keyUsersCollector struct {
+	logger log.Logger
+}
+
+type keyUsersEntry struct {
+	uid  int64
+	name string
+}
+
+func init() {
+	registerCollector("key_users", defaultDisabled, NewKeyUsersCollector)
+}
+
+func NewKeyUsersCollector(logger log.Logger) (Collector, error) {
+	return &keyUsersCollector{logger}, nil
+}
+
+func (c *keyUsersCollector) Update(ch chan<- prometheus.Metric) error {
+	keyUsers, err := c.getKeyUsers()
+	if err != nil {
+		return fmt.Errorf("couldn't get key-users: %w", err)
+	}
+	level.Debug(c.logger).Log("msg", "Set node_keys", "keyUsers", keyUsers)
+	for k, v := range keyUsers {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName(namespace, keyUsersSubsystem, k.name),
+				fmt.Sprintf("Key utilization field %s.", k.name),
+				[]string{"uid"}, nil,
+			),
+			prometheus.GaugeValue, v, strconv.FormatInt(int64(k.uid), 10),
+		)
+	}
+	return nil
+}
+
+func (c *keyUsersCollector) getKeyUsers() (map[keyUsersEntry]float64, error) {
+	file, err := os.Open(procFilePath("key-users"))
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	return parseKeyUsers(file)
+}
+
+func parseKeyUsers(r io.Reader) (map[keyUsersEntry]float64, error) {
+	var scanner = bufio.NewScanner(r)
+	var keyUsersInfo = map[keyUsersEntry]float64{}
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		match := keyUsersLineRegex.FindStringSubmatch(line)
+		names := keyUsersLineRegex.SubexpNames()
+		if len(match) != len(names) {
+			return nil, fmt.Errorf("invalid line in key-users: %s", line)
+		}
+		uid, err := strconv.ParseInt(match[1], 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("invalid UID in key-users: %s", match[0])
+		}
+		for i := 2; i < len(names); i++ {
+			v, err := strconv.ParseInt(match[i], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid metric '%s' in key-users: %s", names[i], match[i])
+			}
+			keyUsersInfo[keyUsersEntry{uid, names[i]}] = float64(v)
+		}
+	}
+	return keyUsersInfo, scanner.Err()
+}

collector/key_users_linux_test.go

@@ -0,0 +1,79 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux && !nokey_users
+// +build linux,!nokey_users
+
+package collector
+
+import (
+	"os"
+	"testing"
+)
+
+func TestKeyUsers(t *testing.T) {
+	file, err := os.Open("fixtures/proc/key-users")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	keyUsers, err := parseKeyUsers(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if want, got := 51.0, keyUsers[keyUsersEntry{0, "usage"}]; want != got {
+		t.Errorf("want uid 0 usage %f, got %f", want, got)
+	}
+	if want, got := 50.0, keyUsers[keyUsersEntry{0, "nkeys"}]; want != got {
+		t.Errorf("want uid 0 nkeys %f, got %f", want, got)
+	}
+	if want, got := 50.0, keyUsers[keyUsersEntry{0, "nikeys"}]; want != got {
+		t.Errorf("want uid 0 nikeys %f, got %f", want, got)
+	}
+	if want, got := 44.0, keyUsers[keyUsersEntry{0, "qnkeys"}]; want != got {
+		t.Errorf("want uid 0 qnkeys %f, got %f", want, got)
+	}
+	if want, got := 1000000.0, keyUsers[keyUsersEntry{0, "maxkeys"}]; want != got {
+		t.Errorf("want uid 0 maxkeys %f, got %f", want, got)
+	}
+	if want, got := 883.0, keyUsers[keyUsersEntry{0, "qnbytes"}]; want != got {
+		t.Errorf("want uid 0 qnbytes %f, got %f", want, got)
+	}
+	if want, got := 25000000.0, keyUsers[keyUsersEntry{0, "maxbytes"}]; want != got {
+		t.Errorf("want uid 0 maxbytes %f, got %f", want, got)
+	}
+
+	if want, got := 11.0, keyUsers[keyUsersEntry{1000, "usage"}]; want != got {
+		t.Errorf("want uid 0 usage %f, got %f", want, got)
+	}
+	if want, got := 11.0, keyUsers[keyUsersEntry{1000, "nkeys"}]; want != got {
+		t.Errorf("want uid 0 nkeys %f, got %f", want, got)
+	}
+	if want, got := 12.0, keyUsers[keyUsersEntry{1000, "nikeys"}]; want != got {
+		t.Errorf("want uid 0 nikeys %f, got %f", want, got)
+	}
+	if want, got := 13.0, keyUsers[keyUsersEntry{1000, "qnkeys"}]; want != got {
+		t.Errorf("want uid 0 qnkeys %f, got %f", want, got)
+	}
+	if want, got := 200.0, keyUsers[keyUsersEntry{1000, "maxkeys"}]; want != got {
+		t.Errorf("want uid 0 maxkeys %f, got %f", want, got)
+	}
+	if want, got := 250.0, keyUsers[keyUsersEntry{1000, "qnbytes"}]; want != got {
+		t.Errorf("want uid 0 qnbytes %f, got %f", want, got)
+	}
+	if want, got := 20000.0, keyUsers[keyUsersEntry{1000, "maxbytes"}]; want != got {
+		t.Errorf("want uid 0 maxbytes %f, got %f", want, got)
+	}
+}

end-to-end-test.sh

@@ -24,6 +24,7 @@ enabled_collectors=$(cat << COLLECTORS
   infiniband
   interrupts
   ipvs
+  key_users
   ksmd
   lnstat
   loadavg