From df1b53bee29d79526c74650d2374562c9a17dcc5 Mon Sep 17 00:00:00 2001 From: Remi Jouannet Date: Wed, 24 May 2023 17:23:13 +0200 Subject: [PATCH] softnet: additionals metrics from softnet_data, (#2592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * softnet: additionals metrics from softnet_data, https://github.com/prometheus/procfs/pull/473 --------- Signed-off-by: remi Signed-off-by: Rémi Jouannet --- collector/fixtures/e2e-64k-page-output.txt | 24 ++++++++ collector/fixtures/e2e-output.txt | 24 ++++++++ collector/softnet_linux.go | 64 +++++++++++++++++++--- 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 37fe7435..c6750a53 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3040,18 +3040,42 @@ node_softirqs_total{vector="rcu"} 508444 node_softirqs_total{vector="sched"} 622196 node_softirqs_total{vector="tasklet"} 1.783454e+06 node_softirqs_total{vector="timer"} 1.481983e+06 +# HELP node_softnet_backlog_len Softnet backlog status +# TYPE node_softnet_backlog_len gauge +node_softnet_backlog_len{cpu="0"} 0 +node_softnet_backlog_len{cpu="1"} 0 +node_softnet_backlog_len{cpu="2"} 0 +node_softnet_backlog_len{cpu="3"} 0 +# HELP node_softnet_cpu_collision_total Number of collision occur while obtaining device lock while transmitting +# TYPE node_softnet_cpu_collision_total counter +node_softnet_cpu_collision_total{cpu="0"} 0 +node_softnet_cpu_collision_total{cpu="1"} 0 +node_softnet_cpu_collision_total{cpu="2"} 0 +node_softnet_cpu_collision_total{cpu="3"} 0 # HELP node_softnet_dropped_total Number of dropped packets # TYPE node_softnet_dropped_total counter node_softnet_dropped_total{cpu="0"} 0 node_softnet_dropped_total{cpu="1"} 41 node_softnet_dropped_total{cpu="2"} 0 node_softnet_dropped_total{cpu="3"} 0 +# HELP node_softnet_flow_limit_count_total Number of times flow limit has been reached +# TYPE node_softnet_flow_limit_count_total counter +node_softnet_flow_limit_count_total{cpu="0"} 0 +node_softnet_flow_limit_count_total{cpu="1"} 0 +node_softnet_flow_limit_count_total{cpu="2"} 0 +node_softnet_flow_limit_count_total{cpu="3"} 0 # HELP node_softnet_processed_total Number of processed packets # TYPE node_softnet_processed_total counter node_softnet_processed_total{cpu="0"} 299641 node_softnet_processed_total{cpu="1"} 916354 node_softnet_processed_total{cpu="2"} 5.577791e+06 node_softnet_processed_total{cpu="3"} 3.113785e+06 +# HELP node_softnet_received_rps_total Number of times cpu woken up received_rps +# TYPE node_softnet_received_rps_total counter +node_softnet_received_rps_total{cpu="0"} 0 +node_softnet_received_rps_total{cpu="1"} 0 +node_softnet_received_rps_total{cpu="2"} 0 +node_softnet_received_rps_total{cpu="3"} 0 # HELP node_softnet_times_squeezed_total Number of times processing packets ran out of quota # TYPE node_softnet_times_squeezed_total counter node_softnet_times_squeezed_total{cpu="0"} 1 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 4bb41591..3306975e 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3062,18 +3062,42 @@ node_softirqs_total{vector="rcu"} 508444 node_softirqs_total{vector="sched"} 622196 node_softirqs_total{vector="tasklet"} 1.783454e+06 node_softirqs_total{vector="timer"} 1.481983e+06 +# HELP node_softnet_backlog_len Softnet backlog status +# TYPE node_softnet_backlog_len gauge +node_softnet_backlog_len{cpu="0"} 0 +node_softnet_backlog_len{cpu="1"} 0 +node_softnet_backlog_len{cpu="2"} 0 +node_softnet_backlog_len{cpu="3"} 0 +# HELP node_softnet_cpu_collision_total Number of collision occur while obtaining device lock while transmitting +# TYPE node_softnet_cpu_collision_total counter +node_softnet_cpu_collision_total{cpu="0"} 0 +node_softnet_cpu_collision_total{cpu="1"} 0 +node_softnet_cpu_collision_total{cpu="2"} 0 +node_softnet_cpu_collision_total{cpu="3"} 0 # HELP node_softnet_dropped_total Number of dropped packets # TYPE node_softnet_dropped_total counter node_softnet_dropped_total{cpu="0"} 0 node_softnet_dropped_total{cpu="1"} 41 node_softnet_dropped_total{cpu="2"} 0 node_softnet_dropped_total{cpu="3"} 0 +# HELP node_softnet_flow_limit_count_total Number of times flow limit has been reached +# TYPE node_softnet_flow_limit_count_total counter +node_softnet_flow_limit_count_total{cpu="0"} 0 +node_softnet_flow_limit_count_total{cpu="1"} 0 +node_softnet_flow_limit_count_total{cpu="2"} 0 +node_softnet_flow_limit_count_total{cpu="3"} 0 # HELP node_softnet_processed_total Number of processed packets # TYPE node_softnet_processed_total counter node_softnet_processed_total{cpu="0"} 299641 node_softnet_processed_total{cpu="1"} 916354 node_softnet_processed_total{cpu="2"} 5.577791e+06 node_softnet_processed_total{cpu="3"} 3.113785e+06 +# HELP node_softnet_received_rps_total Number of times cpu woken up received_rps +# TYPE node_softnet_received_rps_total counter +node_softnet_received_rps_total{cpu="0"} 0 +node_softnet_received_rps_total{cpu="1"} 0 +node_softnet_received_rps_total{cpu="2"} 0 +node_softnet_received_rps_total{cpu="3"} 0 # HELP node_softnet_times_squeezed_total Number of times processing packets ran out of quota # TYPE node_softnet_times_squeezed_total counter node_softnet_times_squeezed_total{cpu="0"} 1 diff --git a/collector/softnet_linux.go b/collector/softnet_linux.go index a46b6808..42d47780 100644 --- a/collector/softnet_linux.go +++ b/collector/softnet_linux.go @@ -26,11 +26,15 @@ import ( ) type softnetCollector struct { - fs procfs.FS - processed *prometheus.Desc - dropped *prometheus.Desc - timeSqueezed *prometheus.Desc - logger log.Logger + fs procfs.FS + processed *prometheus.Desc + dropped *prometheus.Desc + timeSqueezed *prometheus.Desc + cpuCollision *prometheus.Desc + receivedRps *prometheus.Desc + flowLimitCount *prometheus.Desc + softnetBacklogLen *prometheus.Desc + logger log.Logger } const ( @@ -65,19 +69,41 @@ func NewSoftnetCollector(logger log.Logger) (Collector, error) { "Number of times processing packets ran out of quota", []string{"cpu"}, nil, ), + cpuCollision: prometheus.NewDesc( + prometheus.BuildFQName(namespace, softnetSubsystem, "cpu_collision_total"), + "Number of collision occur while obtaining device lock while transmitting", + []string{"cpu"}, nil, + ), + receivedRps: prometheus.NewDesc( + prometheus.BuildFQName(namespace, softnetSubsystem, "received_rps_total"), + "Number of times cpu woken up received_rps", + []string{"cpu"}, nil, + ), + flowLimitCount: prometheus.NewDesc( + prometheus.BuildFQName(namespace, softnetSubsystem, "flow_limit_count_total"), + "Number of times flow limit has been reached", + []string{"cpu"}, nil, + ), + softnetBacklogLen: prometheus.NewDesc( + prometheus.BuildFQName(namespace, softnetSubsystem, "backlog_len"), + "Softnet backlog status", + []string{"cpu"}, nil, + ), logger: logger, }, nil } // Update gets parsed softnet statistics using procfs. func (c *softnetCollector) Update(ch chan<- prometheus.Metric) error { + var cpu string + stats, err := c.fs.NetSoftnetStat() if err != nil { return fmt.Errorf("could not get softnet statistics: %w", err) } - for cpuNumber, cpuStats := range stats { - cpu := strconv.Itoa(cpuNumber) + for _, cpuStats := range stats { + cpu = strconv.FormatUint(uint64(cpuStats.Index), 10) ch <- prometheus.MustNewConstMetric( c.processed, @@ -97,6 +123,30 @@ func (c *softnetCollector) Update(ch chan<- prometheus.Metric) error { float64(cpuStats.TimeSqueezed), cpu, ) + ch <- prometheus.MustNewConstMetric( + c.cpuCollision, + prometheus.CounterValue, + float64(cpuStats.CPUCollision), + cpu, + ) + ch <- prometheus.MustNewConstMetric( + c.receivedRps, + prometheus.CounterValue, + float64(cpuStats.ReceivedRps), + cpu, + ) + ch <- prometheus.MustNewConstMetric( + c.flowLimitCount, + prometheus.CounterValue, + float64(cpuStats.FlowLimitCount), + cpu, + ) + ch <- prometheus.MustNewConstMetric( + c.softnetBacklogLen, + prometheus.GaugeValue, + float64(cpuStats.SoftnetBacklogLen), + cpu, + ) } return nil