From eaacb2e3c7505aa48bd11993180b026a14cc5c6e Mon Sep 17 00:00:00 2001 From: Abbey Woodyear <81559466+abbeywoodyear@users.noreply.github.com> Date: Thu, 25 May 2023 14:09:32 +0100 Subject: [PATCH] exposing softirq metrics (#2294) Signed-off-by: abbeywoodyear --- README.md | 1 + collector/fixtures/e2e-64k-page-output.txt | 23 ++++++++ collector/fixtures/e2e-output.txt | 23 ++++++++ collector/fixtures/proc/softirqs | 11 ++++ collector/softirq_linux.go | 68 ++++++++++++++++++++++ collector/softirqs_common.go | 50 ++++++++++++++++ end-to-end-test.sh | 1 + 7 files changed, 177 insertions(+) create mode 100644 collector/fixtures/proc/softirqs create mode 100644 collector/softirq_linux.go create mode 100644 collector/softirqs_common.go diff --git a/README.md b/README.md index d542cc4b..6fdc180d 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel conf processes | Exposes aggregate process statistics from `/proc`. | Linux qdisc | Exposes [queuing discipline](https://en.wikipedia.org/wiki/Network_scheduler#Linux_kernel) statistics | Linux slabinfo | Exposes slab statistics from `/proc/slabinfo`. Note that permission of `/proc/slabinfo` is usually 0400, so set it appropriately. | Linux +softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index c6750a53..760b475e 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -2922,6 +2922,7 @@ node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="selinux"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 +node_scrape_collector_success{collector="softirqs"} 1 node_scrape_collector_success{collector="softnet"} 1 node_scrape_collector_success{collector="stat"} 1 node_scrape_collector_success{collector="sysctl"} 1 @@ -3028,6 +3029,28 @@ node_sockstat_UDP_mem_bytes 0 # HELP node_sockstat_sockets_used Number of IPv4 sockets in use. # TYPE node_sockstat_sockets_used gauge node_sockstat_sockets_used 229 +# HELP node_softirqs_functions_total Softirq counts per CPU. +# TYPE node_softirqs_functions_total counter +node_softirqs_functions_total{cpu="0",type="BLOCK"} 23776 +node_softirqs_functions_total{cpu="0",type="HI"} 7 +node_softirqs_functions_total{cpu="0",type="HRTIMER"} 40 +node_softirqs_functions_total{cpu="0",type="IRQ_POLL"} 0 +node_softirqs_functions_total{cpu="0",type="NET_RX"} 43066 +node_softirqs_functions_total{cpu="0",type="NET_TX"} 2301 +node_softirqs_functions_total{cpu="0",type="RCU"} 155929 +node_softirqs_functions_total{cpu="0",type="SCHED"} 378895 +node_softirqs_functions_total{cpu="0",type="TASKLET"} 372 +node_softirqs_functions_total{cpu="0",type="TIMER"} 424191 +node_softirqs_functions_total{cpu="1",type="BLOCK"} 24115 +node_softirqs_functions_total{cpu="1",type="HI"} 1 +node_softirqs_functions_total{cpu="1",type="HRTIMER"} 346 +node_softirqs_functions_total{cpu="1",type="IRQ_POLL"} 0 +node_softirqs_functions_total{cpu="1",type="NET_RX"} 104508 +node_softirqs_functions_total{cpu="1",type="NET_TX"} 2430 +node_softirqs_functions_total{cpu="1",type="RCU"} 146631 +node_softirqs_functions_total{cpu="1",type="SCHED"} 152852 +node_softirqs_functions_total{cpu="1",type="TASKLET"} 1899 +node_softirqs_functions_total{cpu="1",type="TIMER"} 108342 # HELP node_softirqs_total Number of softirq calls. # TYPE node_softirqs_total counter node_softirqs_total{vector="block"} 186066 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 3306975e..a8d56649 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -2944,6 +2944,7 @@ node_scrape_collector_success{collector="schedstat"} 1 node_scrape_collector_success{collector="selinux"} 1 node_scrape_collector_success{collector="slabinfo"} 1 node_scrape_collector_success{collector="sockstat"} 1 +node_scrape_collector_success{collector="softirqs"} 1 node_scrape_collector_success{collector="softnet"} 1 node_scrape_collector_success{collector="stat"} 1 node_scrape_collector_success{collector="sysctl"} 1 @@ -3050,6 +3051,28 @@ node_sockstat_UDP_mem_bytes 0 # HELP node_sockstat_sockets_used Number of IPv4 sockets in use. # TYPE node_sockstat_sockets_used gauge node_sockstat_sockets_used 229 +# HELP node_softirqs_functions_total Softirq counts per CPU. +# TYPE node_softirqs_functions_total counter +node_softirqs_functions_total{cpu="0",type="BLOCK"} 23776 +node_softirqs_functions_total{cpu="0",type="HI"} 7 +node_softirqs_functions_total{cpu="0",type="HRTIMER"} 40 +node_softirqs_functions_total{cpu="0",type="IRQ_POLL"} 0 +node_softirqs_functions_total{cpu="0",type="NET_RX"} 43066 +node_softirqs_functions_total{cpu="0",type="NET_TX"} 2301 +node_softirqs_functions_total{cpu="0",type="RCU"} 155929 +node_softirqs_functions_total{cpu="0",type="SCHED"} 378895 +node_softirqs_functions_total{cpu="0",type="TASKLET"} 372 +node_softirqs_functions_total{cpu="0",type="TIMER"} 424191 +node_softirqs_functions_total{cpu="1",type="BLOCK"} 24115 +node_softirqs_functions_total{cpu="1",type="HI"} 1 +node_softirqs_functions_total{cpu="1",type="HRTIMER"} 346 +node_softirqs_functions_total{cpu="1",type="IRQ_POLL"} 0 +node_softirqs_functions_total{cpu="1",type="NET_RX"} 104508 +node_softirqs_functions_total{cpu="1",type="NET_TX"} 2430 +node_softirqs_functions_total{cpu="1",type="RCU"} 146631 +node_softirqs_functions_total{cpu="1",type="SCHED"} 152852 +node_softirqs_functions_total{cpu="1",type="TASKLET"} 1899 +node_softirqs_functions_total{cpu="1",type="TIMER"} 108342 # HELP node_softirqs_total Number of softirq calls. # TYPE node_softirqs_total counter node_softirqs_total{vector="block"} 186066 diff --git a/collector/fixtures/proc/softirqs b/collector/fixtures/proc/softirqs new file mode 100644 index 00000000..a1dfef5f --- /dev/null +++ b/collector/fixtures/proc/softirqs @@ -0,0 +1,11 @@ + CPU0 CPU1 + HI: 7 1 + TIMER: 424191 108342 + NET_TX: 2301 2430 + NET_RX: 43066 104508 + BLOCK: 23776 24115 + IRQ_POLL: 0 0 + TASKLET: 372 1899 + SCHED: 378895 152852 + HRTIMER: 40 346 + RCU: 155929 146631 diff --git a/collector/softirq_linux.go b/collector/softirq_linux.go new file mode 100644 index 00000000..702f34bb --- /dev/null +++ b/collector/softirq_linux.go @@ -0,0 +1,68 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nosoftirqs +// +build !nosoftirqs + +package collector + +import ( + "fmt" + "strconv" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + softirqLabelNames = []string{"cpu", "type"} +) + +func (c *softirqsCollector) Update(ch chan<- prometheus.Metric) (err error) { + softirqs, err := c.fs.Softirqs() + if err != nil { + return fmt.Errorf("couldn't get softirqs: %w", err) + } + + for cpuNo, value := range softirqs.Hi { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "HI") + } + for cpuNo, value := range softirqs.Timer { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "TIMER") + } + for cpuNo, value := range softirqs.NetTx { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "NET_TX") + } + for cpuNo, value := range softirqs.NetRx { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "NET_RX") + } + for cpuNo, value := range softirqs.Block { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "BLOCK") + } + for cpuNo, value := range softirqs.IRQPoll { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "IRQ_POLL") + } + for cpuNo, value := range softirqs.Tasklet { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "TASKLET") + } + for cpuNo, value := range softirqs.Sched { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "SCHED") + } + for cpuNo, value := range softirqs.HRTimer { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "HRTIMER") + } + for cpuNo, value := range softirqs.RCU { + ch <- c.desc.mustNewConstMetric(float64(value), strconv.Itoa(cpuNo), "RCU") + } + + return err +} diff --git a/collector/softirqs_common.go b/collector/softirqs_common.go new file mode 100644 index 00000000..08ef780f --- /dev/null +++ b/collector/softirqs_common.go @@ -0,0 +1,50 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !nosoftirqs +// +build linux,!nosoftirqs + +package collector + +import ( + "fmt" + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" +) + +type softirqsCollector struct { + fs procfs.FS + desc typedDesc + logger log.Logger +} + +func init() { + registerCollector("softirqs", defaultDisabled, NewSoftirqsCollector) +} + +// NewSoftirqsCollector returns a new Collector exposing softirq stats. +func NewSoftirqsCollector(logger log.Logger) (Collector, error) { + desc := typedDesc{prometheus.NewDesc( + namespace+"_softirqs_functions_total", + "Softirq counts per CPU.", + softirqLabelNames, nil, + ), prometheus.CounterValue} + + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %w", err) + } + + return &softirqsCollector{fs, desc, logger}, nil +} diff --git a/end-to-end-test.sh b/end-to-end-test.sh index 3ceb0eb2..75559e9c 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -42,6 +42,7 @@ enabled_collectors=$(cat << COLLECTORS selinux slabinfo sockstat + softirqs stat sysctl textfile