From 0c10545e8dfb983f4c013281c4904be27f645de2 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 24 Feb 2025 14:49:48 +0530 Subject: [PATCH] collector/mdadm: Support RAID metrics (#3031) Use `sysfs` for RAID stats. Signed-off-by: Pranshu Srivastava --- collector/fixtures/e2e-64k-page-output.txt | 16 + collector/fixtures/e2e-output.txt | 16 + collector/fixtures/sys.ttar | 587 +++++++++++++++++++++ collector/mdadm_linux.go | 50 +- collector/mdadm_linux_test.go | 294 +++++++++++ 5 files changed, 960 insertions(+), 3 deletions(-) create mode 100644 collector/mdadm_linux_test.go diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index 030fa88e..4ff32889 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -1585,6 +1585,14 @@ node_md_blocks_synced{device="md6"} 1.6775552e+07 node_md_blocks_synced{device="md7"} 7.813735424e+09 node_md_blocks_synced{device="md8"} 1.6775552e+07 node_md_blocks_synced{device="md9"} 0 +# HELP node_md_degraded Number of degraded disks on device. +# TYPE node_md_degraded gauge +node_md_degraded{device="md0"} 0 +node_md_degraded{device="md1"} 0 +node_md_degraded{device="md10"} 0 +node_md_degraded{device="md4"} 0 +node_md_degraded{device="md5"} 1 +node_md_degraded{device="md6"} 1 # HELP node_md_disks Number of active/failed/spare disks of device. # TYPE node_md_disks gauge node_md_disks{device="md0",state="active"} 2 @@ -1657,6 +1665,14 @@ node_md_disks_required{device="md6"} 2 node_md_disks_required{device="md7"} 4 node_md_disks_required{device="md8"} 2 node_md_disks_required{device="md9"} 4 +# HELP node_md_raid_disks Number of raid disks on device. +# TYPE node_md_raid_disks gauge +node_md_raid_disks{device="md0"} 2 +node_md_raid_disks{device="md1"} 2 +node_md_raid_disks{device="md10"} 4 +node_md_raid_disks{device="md4"} 3 +node_md_raid_disks{device="md5"} 3 +node_md_raid_disks{device="md6"} 4 # HELP node_md_state Indicates the state of md-device. # TYPE node_md_state gauge node_md_state{device="md0",state="active"} 1 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 8598f447..66178b2d 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -1607,6 +1607,14 @@ node_md_blocks_synced{device="md6"} 1.6775552e+07 node_md_blocks_synced{device="md7"} 7.813735424e+09 node_md_blocks_synced{device="md8"} 1.6775552e+07 node_md_blocks_synced{device="md9"} 0 +# HELP node_md_degraded Number of degraded disks on device. +# TYPE node_md_degraded gauge +node_md_degraded{device="md0"} 0 +node_md_degraded{device="md1"} 0 +node_md_degraded{device="md10"} 0 +node_md_degraded{device="md4"} 0 +node_md_degraded{device="md5"} 1 +node_md_degraded{device="md6"} 1 # HELP node_md_disks Number of active/failed/spare disks of device. # TYPE node_md_disks gauge node_md_disks{device="md0",state="active"} 2 @@ -1679,6 +1687,14 @@ node_md_disks_required{device="md6"} 2 node_md_disks_required{device="md7"} 4 node_md_disks_required{device="md8"} 2 node_md_disks_required{device="md9"} 4 +# HELP node_md_raid_disks Number of raid disks on device. +# TYPE node_md_raid_disks gauge +node_md_raid_disks{device="md0"} 2 +node_md_raid_disks{device="md1"} 2 +node_md_raid_disks{device="md10"} 4 +node_md_raid_disks{device="md4"} 3 +node_md_raid_disks{device="md5"} 3 +node_md_raid_disks{device="md6"} 4 # HELP node_md_state Indicates the state of md-device. # TYPE node_md_state gauge node_md_state{device="md0",state="active"} 1 diff --git a/collector/fixtures/sys.ttar b/collector/fixtures/sys.ttar index 5af46262..486dfd31 100644 --- a/collector/fixtures/sys.ttar +++ b/collector/fixtures/sys.ttar @@ -239,6 +239,593 @@ Lines: 1 none Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/array_state +Lines: 1 +clean +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/chunk_size +Lines: 1 +524288 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0/md/dev-sdg +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/dev-sdg/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0/md/dev-sdh +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/dev-sdh/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/level +Lines: 1 +raid0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/raid_disks +Lines: 1 +2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md0/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md0/md/uuid +Lines: 1 +155f29ff-1716-4107-b362-52307ef86cac +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/array_state +Lines: 1 +clean +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/chunk_size +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/degraded +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1/md/dev-sdi +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/dev-sdi/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1/md/dev-sdj +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/dev-sdj/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/level +Lines: 1 +raid1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/raid_disks +Lines: 1 +2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md1/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/sync_action +Lines: 1 +idle +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/sync_completed +Lines: 1 +none +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md1/md/uuid +Lines: 1 +0fbf5f2c-add2-43c2-bd78-a4be3ab709ef +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/array_state +Lines: 1 +clean +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/chunk_size +Lines: 1 +524288 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/degraded +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/dev-sdu +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/dev-sdu/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/dev-sdv +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/dev-sdv/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/dev-sdw +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/dev-sdw/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/dev-sdx +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/dev-sdx/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/level +Lines: 1 +raid10 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/raid_disks +Lines: 1 +4 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/rd2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/rd2/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md10/md/rd3 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/rd3/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/sync_action +Lines: 1 +idle +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/sync_completed +Lines: 1 +none +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md10/md/uuid +Lines: 1 +0c15f7e7-b159-4b1f-a5cd-a79b5c04b6f5 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/array_state +Lines: 1 +clean +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/chunk_size +Lines: 1 +524288 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/degraded +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/dev-sdk +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/dev-sdk/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/dev-sdl +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/dev-sdl/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/dev-sdm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/dev-sdm/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/level +Lines: 1 +raid4 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/raid_disks +Lines: 1 +3 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md4/md/rd2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/rd2/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/sync_action +Lines: 1 +idle +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/sync_completed +Lines: 1 +none +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md4/md/uuid +Lines: 1 +67f415d5-2c0c-4b69-8e0d-7e20ef553457 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/array_state +Lines: 1 +clean +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/chunk_size +Lines: 1 +524288 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/degraded +Lines: 1 +1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/dev-sdaa +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/dev-sdaa/state +Lines: 1 +spare +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/dev-sdn +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/dev-sdn/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/dev-sdo +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/dev-sdo/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/dev-sdp +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/dev-sdp/state +Lines: 1 +faulty +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/level +Lines: 1 +raid5 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/raid_disks +Lines: 1 +3 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md5/md/rd2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/rd2/state +Lines: 1 +faulty +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/sync_action +Lines: 1 +idle +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/sync_completed +Lines: 1 +none +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md5/md/uuid +Lines: 1 +7615b98d-f2ba-4d99-bee8-6202d8e130b9 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/array_state +Lines: 1 +active +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/chunk_size +Lines: 1 +524288 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/degraded +Lines: 1 +1 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/dev-sdq +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/dev-sdq/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/dev-sdr +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/dev-sdr/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/dev-sds +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/dev-sds/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/dev-sdt +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/dev-sdt/state +Lines: 1 +spare +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/level +Lines: 1 +raid6 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/metadata_version +Lines: 1 +1.2 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/raid_disks +Lines: 1 +4 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/rd0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/rd0/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/rd1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/rd1/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/rd2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/rd2/state +Lines: 1 +in_sync +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/md6/md/rd3 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/rd3/state +Lines: 1 +spare +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/sync_action +Lines: 1 +recover +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/sync_completed +Lines: 1 +1569888 / 2093056 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/md6/md/uuid +Lines: 1 +5f529b25-6efd-46e4-99a2-31f6f597be6b +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: sys/bus Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/collector/mdadm_linux.go b/collector/mdadm_linux.go index 5f274ca2..5f76db23 100644 --- a/collector/mdadm_linux.go +++ b/collector/mdadm_linux.go @@ -22,6 +22,8 @@ import ( "log/slog" "os" + "github.com/prometheus/procfs/sysfs" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" ) @@ -98,17 +100,30 @@ var ( []string{"device"}, nil, ) + + mdraidDisks = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "md", "raid_disks"), + "Number of raid disks on device.", + []string{"device"}, + nil, + ) + + mdraidDegradedDisksDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "md", "degraded"), + "Number of degraded disks on device.", + []string{"device"}, + nil, + ) ) func (c *mdadmCollector) Update(ch chan<- prometheus.Metric) error { - fs, err := procfs.NewFS(*procPath) + procFS, err := procfs.NewFS(*procPath) if err != nil { return fmt.Errorf("failed to open procfs: %w", err) } - mdStats, err := fs.MDStat() - + mdStats, err := procFS.MDStat() if err != nil { if errors.Is(err, os.ErrNotExist) { c.logger.Debug("Not collecting mdstat, file does not exist", "file", *procPath) @@ -201,5 +216,34 @@ func (c *mdadmCollector) Update(ch chan<- prometheus.Metric) error { ) } + sysFS, err := sysfs.NewFS(*sysPath) + if err != nil { + return fmt.Errorf("failed to open sysfs: %w", err) + } + mdraids, err := sysFS.Mdraids() + if err != nil { + if errors.Is(err, os.ErrNotExist) { + c.logger.Debug("Not collecting mdraids, file does not exist", "file", *sysPath) + return ErrNoData + } + + return fmt.Errorf("error parsing mdraids: %w", err) + } + + for _, mdraid := range mdraids { + ch <- prometheus.MustNewConstMetric( + mdraidDisks, + prometheus.GaugeValue, + float64(mdraid.Disks), + mdraid.Device, + ) + ch <- prometheus.MustNewConstMetric( + mdraidDegradedDisksDesc, + prometheus.GaugeValue, + float64(mdraid.DegradedDisks), + mdraid.Device, + ) + } + return nil } diff --git a/collector/mdadm_linux_test.go b/collector/mdadm_linux_test.go new file mode 100644 index 00000000..f125d533 --- /dev/null +++ b/collector/mdadm_linux_test.go @@ -0,0 +1,294 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nomdadm +// +build !nomdadm + +package collector + +import ( + "log/slog" + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +type testMdadmCollector struct { + mc Collector +} + +func (c testMdadmCollector) Collect(ch chan<- prometheus.Metric) { + c.mc.Update(ch) +} + +func (c testMdadmCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(c, ch) +} + +func NewTestMdadmCollector(logger *slog.Logger) (prometheus.Collector, error) { + mc, err := NewMdadmCollector(logger) + if err != nil { + return testMdadmCollector{}, err + } + return &testMdadmCollector{mc}, nil +} + +func TestMdadmStats(t *testing.T) { + *sysPath = "fixtures/sys" + *procPath = "fixtures/proc" + testcase := `# HELP node_md_blocks Total number of blocks on device. + # TYPE node_md_blocks gauge + node_md_blocks{device="md0"} 248896 + node_md_blocks{device="md00"} 4.186624e+06 + node_md_blocks{device="md10"} 3.14159265e+08 + node_md_blocks{device="md101"} 322560 + node_md_blocks{device="md11"} 4.190208e+06 + node_md_blocks{device="md12"} 3.886394368e+09 + node_md_blocks{device="md120"} 2.095104e+06 + node_md_blocks{device="md126"} 1.855870976e+09 + node_md_blocks{device="md127"} 3.12319552e+08 + node_md_blocks{device="md201"} 1.993728e+06 + node_md_blocks{device="md219"} 7932 + node_md_blocks{device="md3"} 5.853468288e+09 + node_md_blocks{device="md4"} 4.883648e+06 + node_md_blocks{device="md6"} 1.95310144e+08 + node_md_blocks{device="md7"} 7.813735424e+09 + node_md_blocks{device="md8"} 1.95310144e+08 + node_md_blocks{device="md9"} 523968 + # HELP node_md_blocks_synced Number of blocks synced on device. + # TYPE node_md_blocks_synced gauge + node_md_blocks_synced{device="md0"} 248896 + node_md_blocks_synced{device="md00"} 4.186624e+06 + node_md_blocks_synced{device="md10"} 3.14159265e+08 + node_md_blocks_synced{device="md101"} 322560 + node_md_blocks_synced{device="md11"} 0 + node_md_blocks_synced{device="md12"} 3.886394368e+09 + node_md_blocks_synced{device="md120"} 2.095104e+06 + node_md_blocks_synced{device="md126"} 1.855870976e+09 + node_md_blocks_synced{device="md127"} 3.12319552e+08 + node_md_blocks_synced{device="md201"} 114176 + node_md_blocks_synced{device="md219"} 7932 + node_md_blocks_synced{device="md3"} 5.853468288e+09 + node_md_blocks_synced{device="md4"} 4.883648e+06 + node_md_blocks_synced{device="md6"} 1.6775552e+07 + node_md_blocks_synced{device="md7"} 7.813735424e+09 + node_md_blocks_synced{device="md8"} 1.6775552e+07 + node_md_blocks_synced{device="md9"} 0 + # HELP node_md_degraded Number of degraded disks on device. + # TYPE node_md_degraded gauge + node_md_degraded{device="md0"} 0 + node_md_degraded{device="md1"} 0 + node_md_degraded{device="md10"} 0 + node_md_degraded{device="md4"} 0 + node_md_degraded{device="md5"} 1 + node_md_degraded{device="md6"} 1 + # HELP node_md_disks Number of active/failed/spare disks of device. + # TYPE node_md_disks gauge + node_md_disks{device="md0",state="active"} 2 + node_md_disks{device="md0",state="failed"} 0 + node_md_disks{device="md0",state="spare"} 0 + node_md_disks{device="md00",state="active"} 1 + node_md_disks{device="md00",state="failed"} 0 + node_md_disks{device="md00",state="spare"} 0 + node_md_disks{device="md10",state="active"} 2 + node_md_disks{device="md10",state="failed"} 0 + node_md_disks{device="md10",state="spare"} 0 + node_md_disks{device="md101",state="active"} 3 + node_md_disks{device="md101",state="failed"} 0 + node_md_disks{device="md101",state="spare"} 0 + node_md_disks{device="md11",state="active"} 2 + node_md_disks{device="md11",state="failed"} 1 + node_md_disks{device="md11",state="spare"} 2 + node_md_disks{device="md12",state="active"} 2 + node_md_disks{device="md12",state="failed"} 0 + node_md_disks{device="md12",state="spare"} 0 + node_md_disks{device="md120",state="active"} 2 + node_md_disks{device="md120",state="failed"} 0 + node_md_disks{device="md120",state="spare"} 0 + node_md_disks{device="md126",state="active"} 2 + node_md_disks{device="md126",state="failed"} 0 + node_md_disks{device="md126",state="spare"} 0 + node_md_disks{device="md127",state="active"} 2 + node_md_disks{device="md127",state="failed"} 0 + node_md_disks{device="md127",state="spare"} 0 + node_md_disks{device="md201",state="active"} 2 + node_md_disks{device="md201",state="failed"} 0 + node_md_disks{device="md201",state="spare"} 0 + node_md_disks{device="md219",state="active"} 0 + node_md_disks{device="md219",state="failed"} 0 + node_md_disks{device="md219",state="spare"} 3 + node_md_disks{device="md3",state="active"} 8 + node_md_disks{device="md3",state="failed"} 0 + node_md_disks{device="md3",state="spare"} 2 + node_md_disks{device="md4",state="active"} 0 + node_md_disks{device="md4",state="failed"} 1 + node_md_disks{device="md4",state="spare"} 1 + node_md_disks{device="md6",state="active"} 1 + node_md_disks{device="md6",state="failed"} 1 + node_md_disks{device="md6",state="spare"} 1 + node_md_disks{device="md7",state="active"} 3 + node_md_disks{device="md7",state="failed"} 1 + node_md_disks{device="md7",state="spare"} 0 + node_md_disks{device="md8",state="active"} 2 + node_md_disks{device="md8",state="failed"} 0 + node_md_disks{device="md8",state="spare"} 2 + node_md_disks{device="md9",state="active"} 4 + node_md_disks{device="md9",state="failed"} 2 + node_md_disks{device="md9",state="spare"} 1 + # HELP node_md_disks_required Total number of disks of device. + # TYPE node_md_disks_required gauge + node_md_disks_required{device="md0"} 2 + node_md_disks_required{device="md00"} 1 + node_md_disks_required{device="md10"} 2 + node_md_disks_required{device="md101"} 3 + node_md_disks_required{device="md11"} 2 + node_md_disks_required{device="md12"} 2 + node_md_disks_required{device="md120"} 2 + node_md_disks_required{device="md126"} 2 + node_md_disks_required{device="md127"} 2 + node_md_disks_required{device="md201"} 2 + node_md_disks_required{device="md219"} 0 + node_md_disks_required{device="md3"} 8 + node_md_disks_required{device="md4"} 0 + node_md_disks_required{device="md6"} 2 + node_md_disks_required{device="md7"} 4 + node_md_disks_required{device="md8"} 2 + node_md_disks_required{device="md9"} 4 + # HELP node_md_raid_disks Number of raid disks on device. + # TYPE node_md_raid_disks gauge + node_md_raid_disks{device="md0"} 2 + node_md_raid_disks{device="md1"} 2 + node_md_raid_disks{device="md10"} 4 + node_md_raid_disks{device="md4"} 3 + node_md_raid_disks{device="md5"} 3 + node_md_raid_disks{device="md6"} 4 + # HELP node_md_state Indicates the state of md-device. + # TYPE node_md_state gauge + node_md_state{device="md0",state="active"} 1 + node_md_state{device="md0",state="check"} 0 + node_md_state{device="md0",state="inactive"} 0 + node_md_state{device="md0",state="recovering"} 0 + node_md_state{device="md0",state="resync"} 0 + node_md_state{device="md00",state="active"} 1 + node_md_state{device="md00",state="check"} 0 + node_md_state{device="md00",state="inactive"} 0 + node_md_state{device="md00",state="recovering"} 0 + node_md_state{device="md00",state="resync"} 0 + node_md_state{device="md10",state="active"} 1 + node_md_state{device="md10",state="check"} 0 + node_md_state{device="md10",state="inactive"} 0 + node_md_state{device="md10",state="recovering"} 0 + node_md_state{device="md10",state="resync"} 0 + node_md_state{device="md101",state="active"} 1 + node_md_state{device="md101",state="check"} 0 + node_md_state{device="md101",state="inactive"} 0 + node_md_state{device="md101",state="recovering"} 0 + node_md_state{device="md101",state="resync"} 0 + node_md_state{device="md11",state="active"} 0 + node_md_state{device="md11",state="check"} 0 + node_md_state{device="md11",state="inactive"} 0 + node_md_state{device="md11",state="recovering"} 0 + node_md_state{device="md11",state="resync"} 1 + node_md_state{device="md12",state="active"} 1 + node_md_state{device="md12",state="check"} 0 + node_md_state{device="md12",state="inactive"} 0 + node_md_state{device="md12",state="recovering"} 0 + node_md_state{device="md12",state="resync"} 0 + node_md_state{device="md120",state="active"} 1 + node_md_state{device="md120",state="check"} 0 + node_md_state{device="md120",state="inactive"} 0 + node_md_state{device="md120",state="recovering"} 0 + node_md_state{device="md120",state="resync"} 0 + node_md_state{device="md126",state="active"} 1 + node_md_state{device="md126",state="check"} 0 + node_md_state{device="md126",state="inactive"} 0 + node_md_state{device="md126",state="recovering"} 0 + node_md_state{device="md126",state="resync"} 0 + node_md_state{device="md127",state="active"} 1 + node_md_state{device="md127",state="check"} 0 + node_md_state{device="md127",state="inactive"} 0 + node_md_state{device="md127",state="recovering"} 0 + node_md_state{device="md127",state="resync"} 0 + node_md_state{device="md201",state="active"} 0 + node_md_state{device="md201",state="check"} 1 + node_md_state{device="md201",state="inactive"} 0 + node_md_state{device="md201",state="recovering"} 0 + node_md_state{device="md201",state="resync"} 0 + node_md_state{device="md219",state="active"} 0 + node_md_state{device="md219",state="check"} 0 + node_md_state{device="md219",state="inactive"} 1 + node_md_state{device="md219",state="recovering"} 0 + node_md_state{device="md219",state="resync"} 0 + node_md_state{device="md3",state="active"} 1 + node_md_state{device="md3",state="check"} 0 + node_md_state{device="md3",state="inactive"} 0 + node_md_state{device="md3",state="recovering"} 0 + node_md_state{device="md3",state="resync"} 0 + node_md_state{device="md4",state="active"} 0 + node_md_state{device="md4",state="check"} 0 + node_md_state{device="md4",state="inactive"} 1 + node_md_state{device="md4",state="recovering"} 0 + node_md_state{device="md4",state="resync"} 0 + node_md_state{device="md6",state="active"} 0 + node_md_state{device="md6",state="check"} 0 + node_md_state{device="md6",state="inactive"} 0 + node_md_state{device="md6",state="recovering"} 1 + node_md_state{device="md6",state="resync"} 0 + node_md_state{device="md7",state="active"} 1 + node_md_state{device="md7",state="check"} 0 + node_md_state{device="md7",state="inactive"} 0 + node_md_state{device="md7",state="recovering"} 0 + node_md_state{device="md7",state="resync"} 0 + node_md_state{device="md8",state="active"} 0 + node_md_state{device="md8",state="check"} 0 + node_md_state{device="md8",state="inactive"} 0 + node_md_state{device="md8",state="recovering"} 0 + node_md_state{device="md8",state="resync"} 1 + node_md_state{device="md9",state="active"} 0 + node_md_state{device="md9",state="check"} 0 + node_md_state{device="md9",state="inactive"} 0 + node_md_state{device="md9",state="recovering"} 0 + node_md_state{device="md9",state="resync"} 1 +` + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelError, + AddSource: true, + })) + collector, err := NewMdadmCollector(logger) + if err != nil { + panic(err) + } + c, err := NewTestMdadmCollector(logger) + if err != nil { + t.Fatal(err) + } + reg := prometheus.NewRegistry() + reg.MustRegister(c) + + sink := make(chan prometheus.Metric) + go func() { + err := collector.Update(sink) + if err != nil { + panic(err) + } + close(sink) + }() + + err = testutil.GatherAndCompare(reg, strings.NewReader(testcase)) + if err != nil { + t.Fatal(err) + } +}