Add timex collector (#664)

This collector is based on adjtimex(2) system call. The collector returns three values, status if time is synchronised, offset to remote reference, and local clock frequency adjustment. Values are taken from kernel time keeping data structures to avoid getting involved how the synchronisation is implemented. By that I mean one should not care if time is update using ntpd, systemd.timesyncd, ptpd, and so on. Since all time sync implementation will always end up telling to kernel what is the status with time one can simply omit the software in between, and look results of the syncing. As a positive side effect this makes collector very quick and conceptually specific, this does not monitor availability of NTP server, or network in between, or dns resolution, and other unrelated but necessary things. Minimum set of values to keep eye on are the following three: The node_timex_sync_status tells if local clock is in sync with a remote clock. Value is set to zero when synchronisation to a reliable server is lost, or a time sync software is misconfigured. The node_timex_offset_seconds tells how much local clock is off when compared to reference. In case of multiple time references this value is outcome of RFC 5905 adjustment algorithm. Ideally offset should be close to zero, and it depends about use case how large value is acceptable. For example a typical web server is probably fine if offset is about 0.1 or less, but that would not be good enough for mobile phone base station operator. The node_timex_freq tells amount of adjustment to local clock tick frequency. For example if offset is one second and growing the local clock will need instruction to tick quicker. Number value itself is not very important, and occasional small adjustments are fine. When frequency is unusually in stable one can assume quality of time stamps will not be accurate to very far in sub second range. Obviously explaining why local clock frequency behaves like a passenger in roller coaster is different matter. Explanations can vary from system load, to environmental issues such as a machine being physically too hot. Rest of the measurements can help when debugging. If you run a clock server do probably want to collect and keep track of everything. Pull-request: https://github.com/prometheus/node_exporter/pull/664
7 years ago · 3762191e66
parent c169b4b1c5
commit 3762191e66
3 changed files with 197 additions and 1 deletions
--- a/README.md
+++ b/README.md
@ -44,6 +44,7 @@ sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux
 stat | Exposes various statistics from `/proc/stat`. This includes boot time, forks and interrupts. | Linux
 textfile | Exposes statistics read from local disk. The `--collector.textfile.directory` flag must be set. | _any_
 time | Exposes the current system time. | _any_
+timex | Exposes selected adjtimex(2) system call stats. | Linux
 uname | Exposes system information as provided by the uname system call. | Linux
 vmstat | Exposes statistics from `/proc/vmstat`. | Linux
 wifi | Exposes WiFi device and station statistics. | Linux
--- a/collector/timex.go
+++ b/collector/timex.go
@ -0,0 +1,195 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+// +build !notimex
+
+package collector
+
+// #include <sys/timex.h>
+import "C"
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// The system clock is not synchronized to a reliable server.
+	timeError    = C.TIME_ERROR
+	// The timex.Status time resolution bit, 0 = microsecond, 1 = nanoseconds.
+	staNano      = C.STA_NANO
+	// 1 second in
+	nanoSeconds  = 1000000000
+	microSeconds = 1000000
+)
+
+type timexCollector struct {
+	offset,
+	freq,
+	maxerror,
+	esterror,
+	status,
+	constant,
+	tick,
+	ppsfreq,
+	jitter,
+	shift,
+	stabil,
+	jitcnt,
+	calcnt,
+	errcnt,
+	stbcnt,
+	tai,
+	syncStatus typedDesc
+}
+
+func init() {
+	Factories["timex"] = NewTimexCollector
+}
+
+// NewTimexCollector returns a new Collector exposing adjtime(3) stats.
+func NewTimexCollector() (Collector, error) {
+	const subsystem = "timex"
+
+	return &timexCollector{
+		offset: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "offset_seconds"),
+			"Time offset in between local system and reference clock.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		freq: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "frequency_adjustment"),
+			"Local clock frequency adjustment.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		maxerror: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "maxerror_seconds"),
+			"Maximum error in seconds.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		esterror: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "estimated_error_seconds"),
+			"Estimated error in seconds.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		status: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "status"),
+			"Value of the status array bits.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		constant: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "loop_time_constant"),
+			"Phase-locked loop time constant.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		tick: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "tick_seconds"),
+			"Seconds between clock ticks.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		ppsfreq: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_frequency"),
+			"Pulse per second frequency.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		jitter: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_jitter_seconds"),
+			"Pulse per second jitter.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		shift: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_shift_seconds"),
+			"Pulse per second interval duration.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		stabil: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_stability"),
+			"Pulse per second stability.",
+			nil, nil,
+		), prometheus.CounterValue},
+		jitcnt: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_jitter_count"),
+			"Pulse per second count of jitter limit exceeded events.",
+			nil, nil,
+		), prometheus.CounterValue},
+		calcnt: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_calibration_count"),
+			"Pulse per second count of calibration intervals.",
+			nil, nil,
+		), prometheus.CounterValue},
+		errcnt: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_error_count"),
+			"Pulse per second count of calibration errors.",
+			nil, nil,
+		), prometheus.CounterValue},
+		stbcnt: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "pps_stability_exceeded_count"),
+			"Pulse per second count of stability limit exceeded events.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		tai: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "tai_offset"),
+			"International Atomic Time (TAI) offset.",
+			nil, nil,
+		), prometheus.GaugeValue},
+		syncStatus: typedDesc{prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, subsystem, "sync_status"),
+			"Is clock synchronized to a reliable server (1 = yes, 0 = no).",
+			nil, nil,
+		), prometheus.GaugeValue},
+	}, nil
+}
+
+func (c *timexCollector) Update(ch chan<- prometheus.Metric) error {
+	var syncStatus float64
+	var divisor float64
+	var timex = new(syscall.Timex)
+
+	status, err := syscall.Adjtimex(timex)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve adjtimex stats: %v", err)
+	}
+
+	if status == timeError {
+		syncStatus = 0
+	} else {
+		syncStatus = 1
+	}
+	if (timex.Status & staNano) != 0 {
+		divisor = nanoSeconds
+	} else {
+		divisor = microSeconds
+	}
+	ch <- c.syncStatus.mustNewConstMetric(syncStatus)
+	ch <- c.offset.mustNewConstMetric(float64(timex.Offset) / divisor)
+	ch <- c.freq.mustNewConstMetric(float64(timex.Freq))
+	ch <- c.maxerror.mustNewConstMetric(float64(timex.Maxerror) / microSeconds)
+	ch <- c.esterror.mustNewConstMetric(float64(timex.Esterror) / microSeconds)
+	ch <- c.status.mustNewConstMetric(float64(timex.Status))
+	ch <- c.constant.mustNewConstMetric(float64(timex.Constant))
+	ch <- c.tick.mustNewConstMetric(float64(timex.Tick) / microSeconds)
+	ch <- c.ppsfreq.mustNewConstMetric(float64(timex.Ppsfreq))
+	ch <- c.jitter.mustNewConstMetric(float64(timex.Jitter) / divisor)
+	ch <- c.shift.mustNewConstMetric(float64(timex.Shift))
+	ch <- c.stabil.mustNewConstMetric(float64(timex.Stabil))
+	ch <- c.jitcnt.mustNewConstMetric(float64(timex.Jitcnt))
+	ch <- c.calcnt.mustNewConstMetric(float64(timex.Calcnt))
+	ch <- c.errcnt.mustNewConstMetric(float64(timex.Errcnt))
+	ch <- c.stbcnt.mustNewConstMetric(float64(timex.Stbcnt))
+	ch <- c.tai.mustNewConstMetric(float64(timex.Tai))
+
+	return nil
+}
--- a/node_exporter.go
+++ b/node_exporter.go
@ -31,7 +31,7 @@ import (
 )

 const (
-	defaultCollectors = "arp,bcache,conntrack,cpu,diskstats,entropy,edac,exec,filefd,filesystem,hwmon,infiniband,ipvs,loadavg,mdadm,meminfo,netdev,netstat,sockstat,stat,textfile,time,uname,vmstat,wifi,xfs,zfs"
+	defaultCollectors = "arp,bcache,conntrack,cpu,diskstats,entropy,edac,exec,filefd,filesystem,hwmon,infiniband,ipvs,loadavg,mdadm,meminfo,netdev,netstat,sockstat,stat,textfile,time,timex,uname,vmstat,wifi,xfs,zfs"
 )

 var (