From 7151501fca4cc9def82f5ec301da8e97c3ca8288 Mon Sep 17 00:00:00 2001 From: Hunter Long Date: Thu, 5 Mar 2020 00:27:51 -0800 Subject: [PATCH] vue --- handlers/export.go | 16 ++--- handlers/prometheus.go | 130 +++++++++++++++++++++++++++++++--- types/notifications/struct.go | 6 ++ types/services/database.go | 6 +- types/services/methods.go | 2 +- types/services/routine.go | 14 ++++ types/services/struct.go | 7 ++ utils/metrics.go | 29 ++++++++ utils/utils.go | 10 +++ 9 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 utils/metrics.go diff --git a/handlers/export.go b/handlers/export.go index b4a9b314..87e5c540 100644 --- a/handlers/export.go +++ b/handlers/export.go @@ -28,13 +28,13 @@ import ( // ExportChartsJs renders the charts for the index page type ExportData struct { - Core *core.Core `json:"core"` - Services []*services.Service `json:"services"` - Messages []*messages.Message `json:"messages"` - Checkins []*checkins.Checkin `json:"checkins"` - Users []*users.User `json:"users"` - Groups []*groups.Group `json:"groups"` - Notifiers []core.AllNotifiers `json:"notifiers"` + Core *core.Core `json:"core"` + Services map[int64]*services.Service `json:"services"` + Messages []*messages.Message `json:"messages"` + Checkins []*checkins.Checkin `json:"checkins"` + Users []*users.User `json:"users"` + Groups []*groups.Group `json:"groups"` + Notifiers []core.AllNotifiers `json:"notifiers"` } // ExportSettings will export a JSON file containing all of the settings below: @@ -51,7 +51,7 @@ func ExportSettings() ([]byte, error) { //Notifiers: notifications.All(), Checkins: checkins.All(), Users: users.All(), - Services: services.All(), + Services: services.Services(), Groups: groups.All(), Messages: messages.All(), } diff --git a/handlers/prometheus.go b/handlers/prometheus.go index 3e1ce7b2..1f474e0d 100644 --- a/handlers/prometheus.go +++ b/handlers/prometheus.go @@ -18,9 +18,14 @@ package handlers import ( "fmt" "github.com/hunterlong/statping/types/failures" + "github.com/hunterlong/statping/types/notifications" "github.com/hunterlong/statping/types/services" + "github.com/hunterlong/statping/utils" "net/http" + "runtime" + "strconv" "strings" + "time" ) // @@ -33,25 +38,128 @@ import ( // - targets: ['statping:8080'] // +var ( + prefix string + promValues []string + httpRequests int64 + httpBytesIn int64 +) + +func bToMb(b uint64) uint64 { + return b / 1024 / 1024 +} + +func hex2int(hexStr string) uint64 { + cleaned := strings.Replace(hexStr, "0x", "", -1) + result, _ := strconv.ParseUint(cleaned, 16, 64) + return uint64(result) +} + func prometheusHandler(w http.ResponseWriter, r *http.Request) { - metrics := []string{} + promValues = []string{} + prefix = utils.Getenv("PREFIX", "").(string) + if prefix != "" { + prefix = prefix + "_" + } + + secondsOnline := time.Now().Sub(utils.StartTime).Seconds() allFails := failures.All() - system := fmt.Sprintf("statping_total_failures %v\n", allFails) - system += fmt.Sprintf("statping_total_services %v", len(services.All())) - metrics = append(metrics, system) + + var m runtime.MemStats + runtime.ReadMemStats(&m) + + httpMetrics := utils.GetHttpMetrics() + + promValues = append(promValues, "# Statping Prometheus Exporter") + + PrometheusComment("Statping Totals") + PrometheusKeyValue("total_failures", len(allFails)) + PrometheusKeyValue("total_services", len(services.Services())) + PrometheusKeyValue("seconds_online", secondsOnline) + + if secondsOnline < 5 { + return + } + for _, ser := range services.All() { online := 1 if !ser.Online { online = 0 } - met := fmt.Sprintf("statping_service_failures{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, ser.AllFailures().Count()) - met += fmt.Sprintf("statping_service_latency{id=\"%v\" name=\"%v\"} %0.0f\n", ser.Id, ser.Name, (ser.Latency * 100)) - met += fmt.Sprintf("statping_service_online{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, online) - met += fmt.Sprintf("statping_service_status_code{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, ser.LastStatusCode) - met += fmt.Sprintf("statping_service_response_length{id=\"%v\" name=\"%v\"} %v", ser.Id, ser.Name, len([]byte(ser.LastResponse))) - metrics = append(metrics, met) + id := ser.Id + name := ser.Name + + PrometheusComment(fmt.Sprintf("Service #%d '%s':", ser.Id, ser.Name)) + PrometheusExportKey("service_failures", id, name, ser.AllFailures().Count()) + PrometheusExportKey("service_latency", id, name, ser.Latency*100) + PrometheusExportKey("service_online", id, name, online) + PrometheusExportKey("service_status_code", id, name, ser.LastStatusCode) + PrometheusExportKey("service_response_length", id, name, len([]byte(ser.LastResponse))) + PrometheusExportKey("service_ping_time", id, name, ser.PingTime) + PrometheusExportKey("service_last_latency", id, name, ser.LastLatency) + PrometheusExportKey("service_last_lookup", id, name, ser.LastLookupTime) + PrometheusExportKey("service_last_check", id, name, time.Now().Sub(ser.LastCheck).Milliseconds()) + //PrometheusExportKey("service_online_seconds", id, name, ser.SecondsOnline) + //PrometheusExportKey("service_offline_seconds", id, name, ser.SecondsOffline) + } - output := strings.Join(metrics, "\n") + + for _, notif := range notifications.All() { + PrometheusComment(fmt.Sprintf("Notifier %s:", notif.Method)) + PrometheusExportKey("notifier_on_success", notif.Id, notif.Method, 0) + PrometheusExportKey("notifier_on_failure", notif.Id, notif.Method, 0) + } + + PrometheusComment("HTTP Metrics") + PrometheusKeyValue("http_errors", httpMetrics.Errors) + PrometheusKeyValue("http_requests", httpMetrics.Requests) + PrometheusKeyValue("http_bytes", httpMetrics.Bytes) + PrometheusKeyValue("http_request_milliseconds", httpMetrics.Milliseconds) + + // https://golang.org/pkg/runtime/#MemStats + PrometheusComment("Golang Metrics") + PrometheusKeyValue("go_heap_allocated", m.Alloc) + PrometheusKeyValue("go_total_allocated", m.TotalAlloc) + PrometheusKeyValue("go_heap_in_use", m.HeapInuse) + PrometheusKeyValue("go_heap_objects", m.HeapObjects) + PrometheusKeyValue("go_heap_idle", m.HeapIdle) + PrometheusKeyValue("go_heap_released", m.HeapReleased) + PrometheusKeyValue("go_heap_frees", m.Frees) + PrometheusKeyValue("go_lookups", m.Lookups) + PrometheusKeyValue("go_system", m.Sys) + PrometheusKeyValue("go_number_gc", m.NumGC) + PrometheusKeyValue("go_number_gc_forced", m.NumForcedGC) + PrometheusKeyValue("go_goroutines", runtime.NumGoroutine()) + + output := strings.Join(promValues, "\n") w.WriteHeader(http.StatusOK) w.Write([]byte(output)) } + +func PrometheusKeyValue(keyName string, value interface{}) { + val := promValue(value) + prom := fmt.Sprintf("%sstatping_%s %s", prefix, keyName, val) + promValues = append(promValues, prom) +} + +func PrometheusExportKey(keyName string, id int64, name string, value interface{}) { + val := promValue(value) + prom := fmt.Sprintf("%sstatping_%s{id=\"%d\" name=\"%s\"} %s", prefix, keyName, id, name, val) + promValues = append(promValues, prom) +} + +func PrometheusComment(comment string) { + prom := fmt.Sprintf("\n# %v", comment) + promValues = append(promValues, prom) +} + +func promValue(val interface{}) string { + var newVal string + switch v := val.(type) { + case float64: + newVal = fmt.Sprintf("%.4f", v) + default: + newVal = fmt.Sprintf("%v", v) + } + return newVal +} diff --git a/types/notifications/struct.go b/types/notifications/struct.go index ecc320ac..b102d501 100644 --- a/types/notifications/struct.go +++ b/types/notifications/struct.go @@ -63,9 +63,15 @@ type Notification struct { Running chan bool `gorm:"-" json:"-"` testable bool `gorm:"-" json:"testable"` + Hits notificationHits Notifier } +type notificationHits struct { + onSuccess int64 `gorm:"-" json:"-"` + onFailure int64 `gorm:"-" json:"-"` +} + // QueueData is the struct for the messaging queue with service type QueueData struct { Id string diff --git a/types/services/database.go b/types/services/database.go index d743e5a7..1621228e 100644 --- a/types/services/database.go +++ b/types/services/database.go @@ -21,12 +21,16 @@ func Find(id int64) (*Service, error) { return srv, nil } -func All() []*Service { +func all() []*Service { var services []*Service DB().Find(&services) return services } +func All() map[int64]*Service { + return allServices +} + func (s *Service) Create() error { err := DB().Create(&s) if err.Error() != nil { diff --git a/types/services/methods.go b/types/services/methods.go index 900d0e74..87dfa087 100644 --- a/types/services/methods.go +++ b/types/services/methods.go @@ -66,7 +66,7 @@ func SelectAllServices(start bool) (map[int64]*Service, error) { return allServices, nil } - for _, s := range All() { + for _, s := range all() { allServices[s.Id] = s diff --git a/types/services/routine.go b/types/services/routine.go index 4c9d094f..fe5f9118 100644 --- a/types/services/routine.go +++ b/types/services/routine.go @@ -87,6 +87,8 @@ func isIPv6(address string) bool { // checkIcmp will send a ICMP ping packet to the service func CheckIcmp(s *Service, record bool) *Service { + defer s.updateLastCheck() + p := fastping.NewPinger() resolveIP := "ip4:icmp" if isIPv6(s.Domain) { @@ -113,6 +115,8 @@ func CheckIcmp(s *Service, record bool) *Service { // checkTcp will check a TCP service func CheckTcp(s *Service, record bool) *Service { + defer s.updateLastCheck() + dnsLookup, err := dnsCheck(s) if err != nil { if record { @@ -151,8 +155,14 @@ func CheckTcp(s *Service, record bool) *Service { return s } +func (s *Service) updateLastCheck() { + s.LastCheck = time.Now() +} + // checkHttp will check a HTTP service func CheckHttp(s *Service, record bool) *Service { + defer s.updateLastCheck() + dnsLookup, err := dnsCheck(s) if err != nil { if record { @@ -229,12 +239,16 @@ func recordSuccess(s *Service) { } log.WithFields(utils.ToFields(hit, s)).Infoln( fmt.Sprintf("Service #%d '%v' Successful Response: %0.2f ms | Lookup in: %0.2f ms | Online: %v | Interval: %d seconds", s.Id, s.Name, hit.Latency*1000, hit.PingTime*1000, s.Online, s.Interval)) + s.LastLookupTime = int64(hit.PingTime * 1000) + s.LastLatency = int64(hit.Latency * 1000) //notifier.OnSuccess(s) s.SuccessNotified = true } // recordFailure will create a new 'Failure' record in the database for a offline service func recordFailure(s *Service, issue string) { + s.LastOffline = time.Now().UTC() + fail := &failures.Failure{ Service: s.Id, Issue: issue, diff --git a/types/services/struct.go b/types/services/struct.go index 0150caf2..64dfed48 100644 --- a/types/services/struct.go +++ b/types/services/struct.go @@ -58,9 +58,16 @@ type Service struct { SuccessNotified bool `gorm:"-" json:"-"` // Is 'true' if the user has already be informed that the Services now again available LastStatusCode int `gorm:"-" json:"status_code"` LastOnline time.Time `gorm:"-" json:"last_success"` + LastOffline time.Time `gorm:"-" json:"last_error"` Failures []*failures.Failure `gorm:"-" json:"failures,omitempty" scope:"user,admin"` AllCheckins []*checkins.Checkin `gorm:"-" json:"checkins,omitempty" scope:"user,admin"` Stats *Stats `gorm:"-" json:"stats,omitempty"` + LastLookupTime int64 `gorm:"-" json:"-"` + LastLatency int64 `gorm:"-" json:"-"` + LastCheck time.Time `gorm:"-" json:"-"` + + SecondsOnline int64 `gorm:"-" json:"-"` + SecondsOffline int64 `gorm:"-" json:"-"` } type Stats struct { diff --git a/utils/metrics.go b/utils/metrics.go new file mode 100644 index 00000000..f35f0070 --- /dev/null +++ b/utils/metrics.go @@ -0,0 +1,29 @@ +package utils + +import "time" + +func init() { + httpMetric = new(Metrics) +} + +var ( + httpMetric *Metrics + StartTime = time.Now() +) + +type Metrics struct { + Requests int64 + Errors int64 + Bytes int64 + Milliseconds int64 + OnlineTime time.Time +} + +func (h *Metrics) Reset() { + httpMetric = new(Metrics) +} + +func GetHttpMetrics() *Metrics { + defer httpMetric.Reset() + return httpMetric +} diff --git a/utils/utils.go b/utils/utils.go index a884aa98..784d4a32 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -374,7 +374,9 @@ func SaveFile(filename string, data []byte) error { func HttpRequest(url, method string, content interface{}, headers []string, body io.Reader, timeout time.Duration, verifySSL bool) ([]byte, *http.Response, error) { var err error var req *http.Request + t1 := time.Now() if req, err = http.NewRequest(method, url, body); err != nil { + httpMetric.Errors++ return nil, nil, err } req.Header.Set("User-Agent", "Statping") @@ -424,10 +426,18 @@ func HttpRequest(url, method string, content interface{}, headers []string, body } if resp, err = client.Do(req); err != nil { + httpMetric.Errors++ return nil, resp, err } defer resp.Body.Close() contents, err := ioutil.ReadAll(resp.Body) + + // record HTTP metrics + t2 := time.Now().Sub(t1).Milliseconds() + httpMetric.Requests++ + httpMetric.Milliseconds += t2 / httpMetric.Requests + httpMetric.Bytes += int64(len(contents)) + return contents, resp, err }