From b25273fac00c7318c970a7330b1204d2630e6f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Knecht?= Date: Fri, 9 Jul 2021 11:44:03 +0200 Subject: [PATCH] collector/netdev_*: Add detailed interface stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux, we get more detailed interface statistics from netlink than we did from `/proc/net/dev`. This commit adds a new flag (`--collector.netdev.enable-detailed-metrics`) to expose those statistics under new (incompatible) metric names. When enabled, the metric names are also changed on Darwin and BSD platforms to keep everything consistent, but it doesn't provide more detailed statistics on those platforms. The old metrics can be derived from the new ones using the following rules ([dev_seq_printf_stats]): - `receive_errs` = `receive_errors` - `receive_drop` = `receive_dropped` + `receive_missed_errors` - `receive_fifo` = `receive_fifo_errors` - `receive_frame` = `receive_length_errors` + `receive_over_errors` + `receive_crc_errors` + `receive_frame_errors` - `receive_multicast` = `multicast` - `transmit_errs` = `transmit_errors` - `transmit_drop` = `transmit_dropped` - `transmit_fifo` = `transmit_fifo_errors` - `transmit_colls` = `collisions` - `transmit_carrier` = `transmit_aborted_errors` + `transmit_carrier_errors` + `transmit_heartbeat_errors` + `transmit_window_errors` [dev_seq_printf_stats]: https://github.com/torvalds/linux/blob/master/net/core/net-procfs.c#L75-L97 Signed-off-by: Benoît Knecht --- collector/netdev_bsd.go | 8 ++-- collector/netdev_common.go | 55 ++++++++++++++++++++++++++ collector/netdev_darwin.go | 8 +++- collector/netdev_linux.go | 43 +++++++++++++-------- collector/netdev_linux_test.go | 64 ++++++++++++++++++++++++++++++- collector/netdev_openbsd.go | 10 +++-- collector/netdev_openbsd_amd64.go | 10 +++-- 7 files changed, 169 insertions(+), 29 deletions(-) diff --git a/collector/netdev_bsd.go b/collector/netdev_bsd.go index dc4d1a4f..691bbec4 100644 --- a/collector/netdev_bsd.go +++ b/collector/netdev_bsd.go @@ -59,14 +59,14 @@ func getNetDevStats(filter *deviceFilter, logger log.Logger) (netDevStats, error netDev[dev] = map[string]uint64{ "receive_packets": uint64(data.ifi_ipackets), "transmit_packets": uint64(data.ifi_opackets), - "receive_errs": uint64(data.ifi_ierrors), - "transmit_errs": uint64(data.ifi_oerrors), "receive_bytes": uint64(data.ifi_ibytes), "transmit_bytes": uint64(data.ifi_obytes), + "receive_errors": uint64(data.ifi_ierrors), + "transmit_errors": uint64(data.ifi_oerrors), + "receive_dropped": uint64(data.ifi_iqdrops), + "transmit_dropped": uint64(data.ifi_oqdrops), "receive_multicast": uint64(data.ifi_imcasts), "transmit_multicast": uint64(data.ifi_omcasts), - "receive_drop": uint64(data.ifi_iqdrops), - "transmit_drop": uint64(data.ifi_oqdrops), } } diff --git a/collector/netdev_common.go b/collector/netdev_common.go index 19b07f8c..0a9b522b 100644 --- a/collector/netdev_common.go +++ b/collector/netdev_common.go @@ -36,6 +36,7 @@ var ( netdevDeviceExclude = kingpin.Flag("collector.netdev.device-exclude", "Regexp of net devices to exclude (mutually exclusive to device-include).").String() oldNetdevDeviceExclude = kingpin.Flag("collector.netdev.device-blacklist", "DEPRECATED: Use collector.netdev.device-exclude").Hidden().String() netdevAddressInfo = kingpin.Flag("collector.netdev.address-info", "Collect address-info for every device").Bool() + netdevDetailedMetrics = kingpin.Flag("collector.netdev.enable-detailed-metrics", "Use (incompatible) metric names that provide more detailed stats on Linux").Bool() ) type netDevCollector struct { @@ -114,6 +115,9 @@ func (c *netDevCollector) Update(ch chan<- prometheus.Metric) error { return fmt.Errorf("couldn't get netstats: %w", err) } for dev, devStats := range netDev { + if !*netdevDetailedMetrics { + legacy(devStats) + } for key, value := range devStats { desc := c.metricDesc(key) ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(value), dev) @@ -184,3 +188,54 @@ func getAddrsInfo(interfaces []net.Interface) []addrInfo { return res } + +// https://github.com/torvalds/linux/blob/master/net/core/net-procfs.c#L75-L97 +func legacy(metrics map[string]uint64) { + if metric, ok := pop(metrics, "receive_errors"); ok { + metrics["receive_errs"] = metric + } + if metric, ok := pop(metrics, "receive_dropped"); ok { + metrics["receive_drop"] = metric + popz(metrics, "receive_missed_errors") + } + if metric, ok := pop(metrics, "receive_fifo_errors"); ok { + metrics["receive_fifo"] = metric + } + if metric, ok := pop(metrics, "receive_frame_errors"); ok { + metrics["receive_frame"] = metric + popz(metrics, "receive_length_errors") + popz(metrics, "receive_over_errors") + popz(metrics, "receive_crc_errors") + } + if metric, ok := pop(metrics, "multicast"); ok { + metrics["receive_multicast"] = metric + } + if metric, ok := pop(metrics, "transmit_errors"); ok { + metrics["transmit_errs"] = metric + } + if metric, ok := pop(metrics, "transmit_dropped"); ok { + metrics["transmit_drop"] = metric + } + if metric, ok := pop(metrics, "transmit_fifo_errors"); ok { + metrics["transmit_fifo"] = metric + } + if metric, ok := pop(metrics, "multicast"); ok { + metrics["receive_multicast"] = metric + } + if metric, ok := pop(metrics, "collisions"); ok { + metrics["transmit_colls"] = metric + } + if metric, ok := pop(metrics, "transmit_carrier_errors"); ok { + metrics["transmit_carrier"] = metric + popz(metrics, "transmit_aborted_errors") + popz(metrics, "transmit_heartbeat_errors") + popz(metrics, "transmit_window_errors") + } +} + +func pop(m map[string]uint64, key string) (uint64, bool) { + value, ok := m[key] + delete(m, key) + return value, ok +} + +func popz(m map[string]uint64, key string) uint64 { + if value, ok := m[key]; ok { + delete(m, key) + return value + } + return 0 +} diff --git a/collector/netdev_darwin.go b/collector/netdev_darwin.go index 2ce3d9df..c08f1f8e 100644 --- a/collector/netdev_darwin.go +++ b/collector/netdev_darwin.go @@ -50,12 +50,15 @@ func getNetDevStats(filter *deviceFilter, logger log.Logger) (netDevStats, error netDev[iface.Name] = map[string]uint64{ "receive_packets": ifaceData.Data.Ipackets, "transmit_packets": ifaceData.Data.Opackets, - "receive_errs": ifaceData.Data.Ierrors, - "transmit_errs": ifaceData.Data.Oerrors, "receive_bytes": ifaceData.Data.Ibytes, "transmit_bytes": ifaceData.Data.Obytes, + "receive_errors": ifaceData.Data.Ierrors, + "transmit_errors": ifaceData.Data.Oerrors, + "receive_dropped": ifaceData.Data.Iqdrops, "receive_multicast": ifaceData.Data.Imcasts, "transmit_multicast": ifaceData.Data.Omcasts, + "collisions": ifaceData.Data.Collisions, + "noproto": ifaceData.Data.Noproto, } } @@ -87,6 +90,7 @@ type ifMsghdr2 struct { Data ifData64 } +// https://github.com/apple/darwin-xnu/blob/main/bsd/net/if_var.h#L199-L231 type ifData64 struct { Type uint8 Typelen uint8 diff --git a/collector/netdev_linux.go b/collector/netdev_linux.go index 5a0d5c34..325d10b7 100644 --- a/collector/netdev_linux.go +++ b/collector/netdev_linux.go @@ -51,24 +51,37 @@ func netlinkStats(links []rtnetlink.LinkMessage, filter *deviceFilter, logger lo } // https://github.com/torvalds/linux/blob/master/include/uapi/linux/if_link.h#L42-L246 - // https://github.com/torvalds/linux/blob/master/net/core/net-procfs.c#L75-L97 metrics[name] = map[string]uint64{ - "receive_packets": stats.RXPackets, - "transmit_packets": stats.TXPackets, - "receive_bytes": stats.RXBytes, - "transmit_bytes": stats.TXBytes, - "receive_errs": stats.RXErrors, - "transmit_errs": stats.TXErrors, - "receive_drop": stats.RXDropped + stats.RXMissedErrors, - "transmit_drop": stats.TXDropped, - "receive_multicast": stats.Multicast, - "transmit_colls": stats.Collisions, - "receive_frame": stats.RXLengthErrors + stats.RXOverErrors + stats.RXCRCErrors + stats.RXFrameErrors, - "receive_fifo": stats.RXFIFOErrors, - "transmit_carrier": stats.TXAbortedErrors + stats.TXCarrierErrors + stats.TXHeartbeatErrors + stats.TXWindowErrors, - "transmit_fifo": stats.TXFIFOErrors, + "receive_packets": stats.RXPackets, + "transmit_packets": stats.TXPackets, + "receive_bytes": stats.RXBytes, + "transmit_bytes": stats.TXBytes, + "receive_errors": stats.RXErrors, + "transmit_errors": stats.TXErrors, + "receive_dropped": stats.RXDropped, + "transmit_dropped": stats.TXDropped, + "multicast": stats.Multicast, + "collisions": stats.Collisions, + + // detailed rx_errors + "receive_length_errors": stats.RXLengthErrors, + "receive_over_errors": stats.RXOverErrors, + "receive_crc_errors": stats.RXCRCErrors, + "receive_frame_errors": stats.RXFrameErrors, + "receive_fifo_errors": stats.RXFIFOErrors, + "receive_missed_errors": stats.RXMissedErrors, + + // detailed tx_errors + "transmit_aborted_errors": stats.TXAbortedErrors, + "transmit_carrier_errors": stats.TXCarrierErrors, + "transmit_fifo_errors": stats.TXFIFOErrors, + "transmit_heartbeat_errors": stats.TXHeartbeatErrors, + "transmit_window_errors": stats.TXWindowErrors, + + // for cslip etc "receive_compressed": stats.RXCompressed, "transmit_compressed": stats.TXCompressed, + "receive_nohandler": stats.RXNoHandler, } } diff --git a/collector/netdev_linux_test.go b/collector/netdev_linux_test.go index 541e88d3..32e3d16b 100644 --- a/collector/netdev_linux_test.go +++ b/collector/netdev_linux_test.go @@ -189,7 +189,7 @@ func TestNetDevStatsIgnore(t *testing.T) { t.Error("want fixture interface ibr10:30 to exist, but it does not") } - if want, got := uint64(72), netStats["💩0"]["receive_multicast"]; want != got { + if want, got := uint64(72), netStats["💩0"]["multicast"]; want != got { t.Error("want fixture interface 💩0 to exist, but it does not") } } @@ -201,7 +201,7 @@ func TestNetDevStatsAccept(t *testing.T) { if want, got := 1, len(netStats); want != got { t.Errorf("want count of devices to be %d, got %d", want, got) } - if want, got := uint64(72), netStats["💩0"]["receive_multicast"]; want != got { + if want, got := uint64(72), netStats["💩0"]["multicast"]; want != got { t.Error("want fixture interface 💩0 to exist, but it does not") } } @@ -230,6 +230,7 @@ func TestNetDevLegacyMetricNames(t *testing.T) { netStats := netlinkStats(links, &filter, log.NewNopLogger()) for dev, devStats := range netStats { + legacy(devStats) for _, name := range expected { if _, ok := devStats[name]; !ok { t.Errorf("metric %s should be defined on interface %s", name, dev) @@ -265,6 +266,8 @@ func TestNetDevLegacyMetricValues(t *testing.T) { t.Error("expected stats for interface enp0s0f0") } + legacy(metrics) + for name, want := range expected { got, ok := metrics[name] if !ok { @@ -276,3 +279,60 @@ func TestNetDevLegacyMetricValues(t *testing.T) { } } } + +func TestNetDevMetricValues(t *testing.T) { + filter := newDeviceFilter("", "") + netStats := netlinkStats(links, &filter, log.NewNopLogger()) + + for _, msg := range links { + device := msg.Attributes.Name + stats := msg.Attributes.Stats64 + + expected := map[string]uint64{ + "receive_packets": stats.RXPackets, + "transmit_packets": stats.TXPackets, + "receive_bytes": stats.RXBytes, + "transmit_bytes": stats.TXBytes, + "receive_errors": stats.RXErrors, + "transmit_errors": stats.TXErrors, + "receive_dropped": stats.RXDropped, + "transmit_dropped": stats.TXDropped, + "multicast": stats.Multicast, + "collisions": stats.Collisions, + + // detailed rx_errors + "receive_length_errors": stats.RXLengthErrors, + "receive_over_errors": stats.RXOverErrors, + "receive_crc_errors": stats.RXCRCErrors, + "receive_frame_errors": stats.RXFrameErrors, + "receive_fifo_errors": stats.RXFIFOErrors, + "receive_missed_errors": stats.RXMissedErrors, + + // detailed tx_errors + "transmit_aborted_errors": stats.TXAbortedErrors, + "transmit_carrier_errors": stats.TXCarrierErrors, + "transmit_fifo_errors": stats.TXFIFOErrors, + "transmit_heartbeat_errors": stats.TXHeartbeatErrors, + "transmit_window_errors": stats.TXWindowErrors, + + // for cslip etc + "receive_compressed": stats.RXCompressed, + "transmit_compressed": stats.TXCompressed, + "receive_nohandler": stats.RXNoHandler, + } + + for name, want := range expected { + devStats, ok := netStats[device] + if !ok { + t.Errorf("expected stats for interface %s", device) + } + got, ok := devStats[name] + if !ok { + t.Errorf("metric %s should be defined on interface %s", name, device) + } + if want != got { + t.Errorf("want %s %d, got %d", name, want, got) + } + } + } +} diff --git a/collector/netdev_openbsd.go b/collector/netdev_openbsd.go index 639b9ec1..5d53678d 100644 --- a/collector/netdev_openbsd.go +++ b/collector/netdev_openbsd.go @@ -53,16 +53,20 @@ func getNetDevStats(filter *deviceFilter, logger log.Logger) (netDevStats, error data := (*C.struct_if_data)(ifa.ifa_data) + // https://github.com/openbsd/src/blob/master/sys/net/if.h#L101-L126 netDev[dev] = map[string]uint64{ "receive_packets": uint64(data.ifi_ipackets), "transmit_packets": uint64(data.ifi_opackets), - "receive_errs": uint64(data.ifi_ierrors), - "transmit_errs": uint64(data.ifi_oerrors), "receive_bytes": uint64(data.ifi_ibytes), "transmit_bytes": uint64(data.ifi_obytes), + "receive_errors": uint64(data.ifi_ierrors), + "transmit_errors": uint64(data.ifi_oerrors), + "receive_dropped": uint64(data.ifi_iqdrops), + "transmit_dropped": uint64(data.ifi_oqdrops), "receive_multicast": uint64(data.ifi_imcasts), "transmit_multicast": uint64(data.ifi_omcasts), - "receive_drop": uint64(data.ifi_iqdrops), + "collisions": uint64(data.ifi_collisions), + "noproto": uint64(data.ifi_noproto), } } diff --git a/collector/netdev_openbsd_amd64.go b/collector/netdev_openbsd_amd64.go index 07f458e4..da8a81f3 100644 --- a/collector/netdev_openbsd_amd64.go +++ b/collector/netdev_openbsd_amd64.go @@ -58,16 +58,20 @@ func getNetDevStats(filter *deviceFilter, logger log.Logger) (netDevStats, error continue } + // https://cs.opensource.google/go/x/sys/+/master:unix/ztypes_openbsd_amd64.go;l=292-316 netDev[dev] = map[string]uint64{ "receive_packets": data.Ipackets, "transmit_packets": data.Opackets, - "receive_errs": data.Ierrors, - "transmit_errs": data.Oerrors, "receive_bytes": data.Ibytes, "transmit_bytes": data.Obytes, + "receive_errors": data.Ierrors, + "transmit_errors": data.Oerrors, + "receive_dropped": data.Iqdrops, + "transmit_dropped": data.Oqdrops, "receive_multicast": data.Imcasts, "transmit_multicast": data.Omcasts, - "receive_drop": data.Iqdrops, + "collisions": data.Collisions, + "noproto": data.Noproto, } } return netDev, nil