From ca5d4056228348a1af4404c2ef99ad8e66d92dd0 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Mon, 17 Feb 2014 17:35:23 +0100 Subject: [PATCH 1/4] Add meminfo metrics --- exporter/native_collector.go | 68 +++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/exporter/native_collector.go b/exporter/native_collector.go index 809b0d09..51c3de74 100644 --- a/exporter/native_collector.go +++ b/exporter/native_collector.go @@ -8,6 +8,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "io" "io/ioutil" + "os" "os/exec" "strconv" "strings" @@ -15,13 +16,15 @@ import ( ) const ( - procLoad = "/proc/loadavg" + procLoad = "/proc/loadavg" + procMemInfo = "/proc/meminfo" ) type nativeCollector struct { loadAvg prometheus.Gauge attributes prometheus.Gauge lastSeen prometheus.Gauge + memInfo prometheus.Gauge name string config config } @@ -39,6 +42,7 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, loadAvg: prometheus.NewGauge(), attributes: prometheus.NewGauge(), lastSeen: prometheus.NewGauge(), + memInfo: prometheus.NewGauge(), } registry.Register( @@ -62,6 +66,13 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, c.attributes, ) + registry.Register( + "node_mem", + "node_exporter: memory details.", + prometheus.NilLabels, + c.memInfo, + ) + return &c, nil } @@ -71,22 +82,35 @@ func (c *nativeCollector) Update() (updates int, err error) { last, err := getSecondsSinceLastLogin() if err != nil { return updates, fmt.Errorf("Couldn't get last seen: %s", err) - } else { - updates++ - debug(c.Name(), "Set node_last_login_seconds: %f", last) - c.lastSeen.Set(nil, last) } + updates++ + debug(c.Name(), "Set node_last_login_seconds: %f", last) + c.lastSeen.Set(nil, last) load, err := getLoad() if err != nil { return updates, fmt.Errorf("Couldn't get load: %s", err) - } else { - updates++ - debug(c.Name(), "Set node_load: %f", load) - c.loadAvg.Set(nil, load) } + updates++ + debug(c.Name(), "Set node_load: %f", load) + c.loadAvg.Set(nil, load) + debug(c.Name(), "Set node_attributes{%v}: 1", c.config.Attributes) c.attributes.Set(c.config.Attributes, 1) + + memInfo, err := getMemInfo() + if err != nil { + return updates, fmt.Errorf("Couldn't get meminfo: %s", err) + } + debug(c.Name(), "Set node_mem: %#v", memInfo) + for k, v := range memInfo { + updates++ + fv, err := strconv.ParseFloat(v, 64) + if err != nil { + return updates, fmt.Errorf("Invalid value in meminfo: %s", err) + } + c.memInfo.Set(map[string]string{"type": k}, fv) + } return updates, err } @@ -152,3 +176,29 @@ func getSecondsSinceLastLogin() (float64, error) { return float64(time.Now().Sub(last).Seconds()), nil } + +func getMemInfo() (map[string]string, error) { + memInfo := map[string]string{} + fh, err := os.Open(procMemInfo) + if err != nil { + return nil, err + } + defer fh.Close() + scanner := bufio.NewScanner(fh) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(string(line)) + key := "" + switch len(parts) { + case 2: // no unit + key = parts[0][:len(parts[0])-1] // remove trailing : from key + case 3: // has unit + key = fmt.Sprintf("%s_%s", parts[0][:len(parts[0])-1], parts[2]) + default: + return nil, fmt.Errorf("Invalid line in %s: %s", procMemInfo, line) + } + memInfo[key] = parts[1] + } + return memInfo, nil + +} From 107f94d90aa50831025ff2843e2b7cdf3547c741 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Mon, 17 Feb 2014 18:36:12 +0100 Subject: [PATCH 2/4] Add interrupts metrics --- exporter/native_collector.go | 76 +++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/exporter/native_collector.go b/exporter/native_collector.go index 51c3de74..5c7cd361 100644 --- a/exporter/native_collector.go +++ b/exporter/native_collector.go @@ -16,8 +16,9 @@ import ( ) const ( - procLoad = "/proc/loadavg" - procMemInfo = "/proc/meminfo" + procLoad = "/proc/loadavg" + procMemInfo = "/proc/meminfo" + procInterrupts = "/proc/interrupts" ) type nativeCollector struct { @@ -25,6 +26,7 @@ type nativeCollector struct { attributes prometheus.Gauge lastSeen prometheus.Gauge memInfo prometheus.Gauge + interrupts prometheus.Counter name string config config } @@ -43,6 +45,7 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, attributes: prometheus.NewGauge(), lastSeen: prometheus.NewGauge(), memInfo: prometheus.NewGauge(), + interrupts: prometheus.NewCounter(), } registry.Register( @@ -73,6 +76,13 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, c.memInfo, ) + registry.Register( + "node_interrupts", + "node_exporter: interrupt details.", + prometheus.NilLabels, + c.interrupts, + ) + return &c, nil } @@ -111,6 +121,27 @@ func (c *nativeCollector) Update() (updates int, err error) { } c.memInfo.Set(map[string]string{"type": k}, fv) } + + interrupts, err := getInterrupts() + if err != nil { + return updates, fmt.Errorf("Couldn't get interrupts: %s", err) + } + for name, interrupt := range interrupts { + for cpuNo, value := range interrupt.values { + updates++ + fv, err := strconv.ParseFloat(value, 64) + if err != nil { + return updates, fmt.Errorf("Invalid value in interrupts: %s", err) + } + labels := map[string]string{ + "CPU": strconv.Itoa(cpuNo), + "type": name, + "info": interrupt.info, + "devices": interrupt.devices, + } + c.interrupts.Set(labels, fv) + } + } return updates, err } @@ -202,3 +233,44 @@ func getMemInfo() (map[string]string, error) { return memInfo, nil } + +type interrupt struct { + info string + devices string + values []string +} + +func getInterrupts() (map[string]interrupt, error) { + interrupts := map[string]interrupt{} + fh, err := os.Open(procInterrupts) + if err != nil { + return nil, err + } + defer fh.Close() + scanner := bufio.NewScanner(fh) + if !scanner.Scan() { + return nil, fmt.Errorf("%s empty", procInterrupts) + } + cpuNum := len(strings.Fields(string(scanner.Text()))) // one header per cpu + + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(string(line)) + if len(parts) < cpuNum+2 { // irq + one column per cpu + details, + continue // we ignore ERR and MIS for now + } + intName := parts[0][:len(parts[0])-1] // remove trailing : + intr := interrupt{ + values: parts[1:cpuNum], + } + + if _, err := strconv.Atoi(intName); err == nil { // numeral interrupt + intr.info = parts[cpuNum+1] + intr.devices = strings.Join(parts[cpuNum+2:], " ") + } else { + intr.info = strings.Join(parts[cpuNum+1:], " ") + } + interrupts[intName] = intr + } + return interrupts, nil +} From 9f17cd31c5c1943e9871ec0eea1a4a1e1385374b Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 18 Feb 2014 11:26:33 +0100 Subject: [PATCH 3/4] Add network metrics --- exporter/native_collector.go | 83 +++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/exporter/native_collector.go b/exporter/native_collector.go index 5c7cd361..5a935b92 100644 --- a/exporter/native_collector.go +++ b/exporter/native_collector.go @@ -19,6 +19,7 @@ const ( procLoad = "/proc/loadavg" procMemInfo = "/proc/meminfo" procInterrupts = "/proc/interrupts" + procNetDev = "/proc/net/dev" ) type nativeCollector struct { @@ -27,6 +28,7 @@ type nativeCollector struct { lastSeen prometheus.Gauge memInfo prometheus.Gauge interrupts prometheus.Counter + netStats prometheus.Counter name string config config } @@ -46,6 +48,7 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, lastSeen: prometheus.NewGauge(), memInfo: prometheus.NewGauge(), interrupts: prometheus.NewCounter(), + netStats: prometheus.NewCounter(), } registry.Register( @@ -83,6 +86,13 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, c.interrupts, ) + registry.Register( + "node_net", + "node_exporter: network stats.", + prometheus.NilLabels, + c.netStats, + ) + return &c, nil } @@ -131,7 +141,7 @@ func (c *nativeCollector) Update() (updates int, err error) { updates++ fv, err := strconv.ParseFloat(value, 64) if err != nil { - return updates, fmt.Errorf("Invalid value in interrupts: %s", err) + return updates, fmt.Errorf("Invalid value in interrupts: %s", fv, err) } labels := map[string]string{ "CPU": strconv.Itoa(cpuNo), @@ -142,6 +152,28 @@ func (c *nativeCollector) Update() (updates int, err error) { c.interrupts.Set(labels, fv) } } + + netStats, err := getNetStats() + if err != nil { + return updates, fmt.Errorf("Couldn't get netstats: %s", err) + } + for direction, devStats := range netStats { + for dev, stats := range devStats { + for t, value := range stats { + updates++ + v, err := strconv.ParseFloat(value, 64) + if err != nil { + return updates, fmt.Errorf("Invalid value %s in interrupts: %s", value, err) + } + labels := map[string]string{ + "device": dev, + "direction": direction, + "type": t, + } + c.netStats.Set(labels, v) + } + } + } return updates, err } @@ -274,3 +306,52 @@ func getInterrupts() (map[string]interrupt, error) { } return interrupts, nil } + +func getNetStats() (map[string]map[string]map[string]string, error) { + netStats := map[string]map[string]map[string]string{} + netStats["transmit"] = map[string]map[string]string{} + netStats["receive"] = map[string]map[string]string{} + fh, err := os.Open(procNetDev) + if err != nil { + return nil, err + } + defer fh.Close() + scanner := bufio.NewScanner(fh) + scanner.Scan() // skip first header + scanner.Scan() + parts := strings.Split(string(scanner.Text()), "|") + if len(parts) != 3 { // interface + receive + transmit + return nil, fmt.Errorf("Invalid header line in %s: %s", + procNetDev, scanner.Text()) + } + header := strings.Fields(parts[1]) + for scanner.Scan() { + parts := strings.Fields(string(scanner.Text())) + if len(parts) != 2*len(header)+1 { + return nil, fmt.Errorf("Invalid line in %s: %s", + procNetDev, scanner.Text()) + } + + dev := parts[0][:len(parts[0])-1] + receive, err := parseNetDevLine(parts[1:len(header)+1], header) + if err != nil { + return nil, err + } + + transmit, err := parseNetDevLine(parts[len(header)+1:], header) + if err != nil { + return nil, err + } + netStats["transmit"][dev] = transmit + netStats["receive"][dev] = receive + } + return netStats, nil +} + +func parseNetDevLine(parts []string, header []string) (map[string]string, error) { + devStats := map[string]string{} + for i, v := range parts { + devStats[header[i]] = v + } + return devStats, nil +} From 2282e77e9f518d0946ce74ea73c936b3046e23af Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 18 Feb 2014 11:54:12 +0100 Subject: [PATCH 4/4] Add disk metrics --- exporter/native_collector.go | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/exporter/native_collector.go b/exporter/native_collector.go index 5a935b92..281a0e09 100644 --- a/exporter/native_collector.go +++ b/exporter/native_collector.go @@ -20,6 +20,17 @@ const ( procMemInfo = "/proc/meminfo" procInterrupts = "/proc/interrupts" procNetDev = "/proc/net/dev" + procDiskStats = "/proc/diskstats" +) + +var ( + diskStatsHeader = []string{ + "reads_completed", "reads_merged", + "sectors_read", "read_time_ms", + "writes_completed", "writes_merged", + "sectors_written", "write_time_ms", + "io_now", "io_time_ms", "io_time_weighted", + } ) type nativeCollector struct { @@ -29,6 +40,7 @@ type nativeCollector struct { memInfo prometheus.Gauge interrupts prometheus.Counter netStats prometheus.Counter + diskStats prometheus.Counter name string config config } @@ -49,6 +61,7 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, memInfo: prometheus.NewGauge(), interrupts: prometheus.NewCounter(), netStats: prometheus.NewCounter(), + diskStats: prometheus.NewCounter(), } registry.Register( @@ -93,6 +106,12 @@ func NewNativeCollector(config config, registry prometheus.Registry) (Collector, c.netStats, ) + registry.Register( + "node_disk", + "node_exporter: disk stats.", + prometheus.NilLabels, + c.diskStats, + ) return &c, nil } @@ -174,6 +193,22 @@ func (c *nativeCollector) Update() (updates int, err error) { } } } + + diskStats, err := getDiskStats() + if err != nil { + return updates, fmt.Errorf("Couldn't get diskstats: %s", err) + } + for dev, stats := range diskStats { + for k, value := range stats { + updates++ + v, err := strconv.ParseFloat(value, 64) + if err != nil { + return updates, fmt.Errorf("Invalid value %s in diskstats: %s", value, err) + } + labels := map[string]string{"device": dev, "type": k} + c.diskStats.Set(labels, v) + } + } return updates, err } @@ -355,3 +390,25 @@ func parseNetDevLine(parts []string, header []string) (map[string]string, error) } return devStats, nil } + +func getDiskStats() (map[string]map[string]string, error) { + diskStats := map[string]map[string]string{} + fh, err := os.Open(procDiskStats) + if err != nil { + return nil, err + } + defer fh.Close() + scanner := bufio.NewScanner(fh) + for scanner.Scan() { + parts := strings.Fields(string(scanner.Text())) + if len(parts) != len(diskStatsHeader)+3 { // we strip major, minor and dev + return nil, fmt.Errorf("Invalid line in %s: %s", procDiskStats, scanner.Text()) + } + dev := parts[2] + diskStats[dev] = map[string]string{} + for i, v := range parts[3:] { + diskStats[dev][diskStatsHeader[i]] = v + } + } + return diskStats, nil +}