From 519203e3d0a0ecb9f6c09f88c27dce3c558a4a87 Mon Sep 17 00:00:00 2001 From: Artur Molchanov Date: Mon, 26 Oct 2020 16:06:22 +0300 Subject: [PATCH] Expose zfs zpool state Create a metric node_zfs_zpool_state. Signed-off-by: Artur Molchanov --- collector/fixtures/e2e-output.txt | 14 ++++ .../fixtures/proc/spl/kstat/zfs/pool1/state | 1 + .../fixtures/proc/spl/kstat/zfs/poolz1/state | 1 + collector/zfs.go | 17 +++++ collector/zfs_linux.go | 66 +++++++++++++++++-- collector/zfs_linux_test.go | 49 ++++++++++++++ 6 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 collector/fixtures/proc/spl/kstat/zfs/pool1/state create mode 100644 collector/fixtures/proc/spl/kstat/zfs/poolz1/state diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index b2ec9118..1f57032a 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3827,6 +3827,20 @@ node_zfs_zpool_rtime{zpool="poolz1"} 9.82909164e+09 # TYPE node_zfs_zpool_rupdate untyped node_zfs_zpool_rupdate{zpool="pool1"} 7.921048984922e+13 node_zfs_zpool_rupdate{zpool="poolz1"} 1.10734831944501e+14 +# HELP node_zfs_zpool_state kstat.zfs.misc.state +# TYPE node_zfs_zpool_state gauge +node_zfs_zpool_state{state="degraded",zpool="pool1"} 0 +node_zfs_zpool_state{state="degraded",zpool="poolz1"} 1 +node_zfs_zpool_state{state="faulted",zpool="pool1"} 0 +node_zfs_zpool_state{state="faulted",zpool="poolz1"} 0 +node_zfs_zpool_state{state="offline",zpool="pool1"} 0 +node_zfs_zpool_state{state="offline",zpool="poolz1"} 0 +node_zfs_zpool_state{state="online",zpool="pool1"} 1 +node_zfs_zpool_state{state="online",zpool="poolz1"} 0 +node_zfs_zpool_state{state="removed",zpool="pool1"} 0 +node_zfs_zpool_state{state="removed",zpool="poolz1"} 0 +node_zfs_zpool_state{state="unavail",zpool="pool1"} 0 +node_zfs_zpool_state{state="unavail",zpool="poolz1"} 0 # HELP node_zfs_zpool_wcnt kstat.zfs.misc.io.wcnt # TYPE node_zfs_zpool_wcnt untyped node_zfs_zpool_wcnt{zpool="pool1"} 0 diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/state b/collector/fixtures/proc/spl/kstat/zfs/pool1/state new file mode 100644 index 00000000..1424865c --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/state @@ -0,0 +1 @@ +ONLINE diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/state b/collector/fixtures/proc/spl/kstat/zfs/poolz1/state new file mode 100644 index 00000000..be5b2ef0 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/state @@ -0,0 +1 @@ +DEGRADED diff --git a/collector/zfs.go b/collector/zfs.go index b530e350..540948dd 100644 --- a/collector/zfs.go +++ b/collector/zfs.go @@ -37,6 +37,7 @@ type zfsCollector struct { linuxProcpathBase string linuxZpoolIoPath string linuxZpoolObjsetPath string + linuxZpoolStatePath string linuxPathMap map[string]string logger log.Logger } @@ -47,6 +48,7 @@ func NewZFSCollector(logger log.Logger) (Collector, error) { linuxProcpathBase: "spl/kstat/zfs", linuxZpoolIoPath: "/*/io", linuxZpoolObjsetPath: "/*/objset-*", + linuxZpoolStatePath: "/*/state", linuxPathMap: map[string]string{ "zfs_abd": "abdstats", "zfs_arc": "arcstats", @@ -132,3 +134,18 @@ func (c *zfsCollector) constPoolObjsetMetric(poolName string, datasetName string datasetName, ) } + +func (c *zfsCollector) constPoolStateMetric(poolName string, stateName string, isActive uint64) prometheus.Metric { + return prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(namespace, "zfs_zpool", "state"), + "kstat.zfs.misc.state", + []string{"zpool", "state"}, + nil, + ), + prometheus.GaugeValue, + float64(isActive), + poolName, + stateName, + ) +} diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go index f55d98a5..01bd6df3 100644 --- a/collector/zfs_linux.go +++ b/collector/zfs_linux.go @@ -41,6 +41,8 @@ const ( // kstatDataString = "7" ) +var zfsPoolStatesName = []string{"online", "degraded", "faulted", "offline", "removed", "unavail"} + func (c *zfsCollector) openProcFile(path string) (*os.File, error) { file, err := os.Open(procFilePath(path)) if err != nil { @@ -97,10 +99,6 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error { return err } - if zpoolObjsetPaths == nil { - return nil - } - for _, zpoolPath := range zpoolObjsetPaths { file, err := os.Open(zpoolPath) if err != nil { @@ -117,6 +115,34 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error { return err } } + + zpoolStatePaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolStatePath))) + if err != nil { + return err + } + + if zpoolStatePaths == nil { + level.Warn(c.logger).Log("msg", "Not found pool state files") + } + + for _, zpoolPath := range zpoolStatePaths { + file, err := os.Open(zpoolPath) + if err != nil { + // this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore + level.Debug(c.logger).Log("msg", "Cannot open file for reading", "path", zpoolPath) + return errZFSNotAvailable + } + + err = c.parsePoolStateFile(file, zpoolPath, func(poolName string, stateName string, isActive uint64) { + ch <- c.constPoolStateMetric(poolName, stateName, isActive) + }) + + file.Close() + if err != nil { + return err + } + } + return nil } @@ -235,3 +261,35 @@ func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, h return scanner.Err() } + +func (c *zfsCollector) parsePoolStateFile(reader io.Reader, zpoolPath string, handler func(string, string, uint64)) error { + scanner := bufio.NewScanner(reader) + scanner.Scan() + + actualStateName, err := scanner.Text(), scanner.Err() + if err != nil { + return err + } + + actualStateName = strings.ToLower(actualStateName) + + zpoolPathElements := strings.Split(zpoolPath, "/") + pathLen := len(zpoolPathElements) + if pathLen < 2 { + return fmt.Errorf("zpool path did not return at least two elements") + } + + zpoolName := zpoolPathElements[pathLen-2] + + for _, stateName := range zfsPoolStatesName { + isActive := uint64(0) + + if actualStateName == stateName { + isActive = 1 + } + + handler(zpoolName, stateName, isActive) + } + + return nil +} diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go index 29d28e8f..8c29acdd 100644 --- a/collector/zfs_linux_test.go +++ b/collector/zfs_linux_test.go @@ -494,3 +494,52 @@ func TestVdevMirrorstatsParsing(t *testing.T) { t.Fatal("VdevMirrorStats parsing handler was not called for some expected sysctls") } } + +func TestPoolStateParsing(t *testing.T) { + zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/state") + if err != nil { + t.Fatal(err) + } + + c := zfsCollector{} + if err != nil { + t.Fatal(err) + } + + handlerCalled := false + for _, zpoolPath := range zpoolPaths { + file, err := os.Open(zpoolPath) + if err != nil { + t.Fatal(err) + } + + err = c.parsePoolStateFile(file, zpoolPath, func(poolName string, stateName string, isActive uint64) { + handlerCalled = true + + if poolName == "pool1" { + if isActive != uint64(1) && stateName == "online" { + t.Fatalf("Incorrect parsed value for online state") + } + if isActive != uint64(0) && stateName != "online" { + t.Fatalf("Incorrect parsed value for online state") + } + } + if poolName == "poolz1" { + if isActive != uint64(1) && stateName == "degraded" { + t.Fatalf("Incorrect parsed value for degraded state") + } + if isActive != uint64(0) && stateName != "degraded" { + t.Fatalf("Incorrect parsed value for degraded state") + } + } + }) + file.Close() + if err != nil { + t.Fatal(err) + } + } + if !handlerCalled { + t.Fatal("Zpool parsing handler was not called for some expected sysctls") + } + +}