pull/663/head
hunterlong 2020-06-14 13:46:22 -07:00
parent 98e80157b5
commit 722ce47977
4 changed files with 64 additions and 34 deletions

View File

@ -1,5 +1,4 @@
{{ define "base" }}
<!DOCTYPE html>
{{ define "base" }}<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">

View File

@ -6,38 +6,6 @@ import (
)
var (
serviceOnline = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_online",
Help: "How many failures occur for a service",
},
[]string{"service"},
)
serviceFailures = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_failures",
Help: "How many failures occur for a service",
},
[]string{"service"},
)
serviceSuccess = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_success",
Help: "How many successful requests for a service",
},
[]string{"service"},
)
serviceLatencyDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "service_request_duration",
Help: "How many successful requests for a service",
},
[]string{"service"},
)
utilsHttpRequestDur = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "http_requests_duration",
@ -60,6 +28,7 @@ func init() {
serviceOnline,
serviceFailures,
serviceSuccess,
serviceStatusCode,
serviceLatencyDuration,
utilsHttpRequestDur,
utilsHttpRequestBytes,
@ -77,6 +46,15 @@ func Histo(method string, value float64, labels ...interface{}) {
}
}
func Gauge(method string, value float64, labels ...interface{}) {
switch method {
case "service":
serviceStatusCode.WithLabelValues(convert(labels)...).Set(value)
case "online":
serviceOnline.WithLabelValues(convert(labels)...).Set(value)
}
}
func Inc(method string, labels ...interface{}) {
switch method {
case "failure":

50
types/metrics/services.go Normal file
View File

@ -0,0 +1,50 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// service is online if set to 1, offline if 0
serviceOnline = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "service_online",
Help: "If service is online",
},
[]string{"service"},
)
// service failures
serviceFailures = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_failures",
Help: "How many failures occur for a service",
},
[]string{"service"},
)
// successful hits for a service
serviceSuccess = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_success",
Help: "How many successful requests for a service",
},
[]string{"service"},
)
// service check latency
serviceLatencyDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "service_latency",
Help: "How many successful requests for a service",
},
[]string{"service"},
)
// http status code for a service
serviceStatusCode = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "service_status_code",
Help: "HTTP Status code for a service",
},
[]string{"service"},
)
)

View File

@ -276,6 +276,7 @@ func CheckHttp(s *Service, record bool) (*Service, error) {
metrics.Histo("latency", utils.Now().Sub(t1).Seconds(), s.Id)
s.LastResponse = string(content)
s.LastStatusCode = res.StatusCode
metrics.Gauge("service", float64(res.StatusCode), s.Id)
if s.Expected.String != "" {
match, err := regexp.MatchString(s.Expected.String, string(content))
@ -321,6 +322,7 @@ func recordSuccess(s *Service) {
s.LastLatency = hit.Latency
sendSuccess(s)
s.SuccessNotified = true
metrics.Gauge("online", 1., s.Id)
}
func AddNotifier(n ServiceNotifier) {
@ -373,6 +375,7 @@ func recordFailure(s *Service, issue string) {
s.SuccessNotified = false
s.DownText = s.DowntimeText()
sendFailure(s, fail)
metrics.Gauge("online", 0., s.Id)
}
func sendFailure(s *Service, f *failures.Failure) {