vue

handlers/export.go

@@ -28,13 +28,13 @@ import (
 // ExportChartsJs renders the charts for the index page
 
 type ExportData struct {
-	Core      *core.Core          `json:"core"`
+	Core      *core.Core                  `json:"core"`
-	Services  []*services.Service `json:"services"`
+	Services  map[int64]*services.Service `json:"services"`
-	Messages  []*messages.Message `json:"messages"`
+	Messages  []*messages.Message         `json:"messages"`
-	Checkins  []*checkins.Checkin `json:"checkins"`
+	Checkins  []*checkins.Checkin         `json:"checkins"`
-	Users     []*users.User       `json:"users"`
+	Users     []*users.User               `json:"users"`
-	Groups    []*groups.Group     `json:"groups"`
+	Groups    []*groups.Group             `json:"groups"`
-	Notifiers []core.AllNotifiers `json:"notifiers"`
+	Notifiers []core.AllNotifiers         `json:"notifiers"`
 }
 
 // ExportSettings will export a JSON file containing all of the settings below:
@@ -51,7 +51,7 @@ func ExportSettings() ([]byte, error) {
 		//Notifiers: notifications.All(),
 		Checkins: checkins.All(),
 		Users:    users.All(),
-		Services: services.All(),
+		Services: services.Services(),
 		Groups:   groups.All(),
 		Messages: messages.All(),
 	}

handlers/prometheus.go

@@ -18,9 +18,14 @@ package handlers
 import (
 	"fmt"
 	"github.com/hunterlong/statping/types/failures"
+	"github.com/hunterlong/statping/types/notifications"
 	"github.com/hunterlong/statping/types/services"
+	"github.com/hunterlong/statping/utils"
 	"net/http"
+	"runtime"
+	"strconv"
 	"strings"
+	"time"
 )
 
 //
@@ -33,25 +38,128 @@ import (
 //			  - targets: ['statping:8080']
 //
 
+var (
+	prefix       string
+	promValues   []string
+	httpRequests int64
+	httpBytesIn  int64
+)
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}
+
+func hex2int(hexStr string) uint64 {
+	cleaned := strings.Replace(hexStr, "0x", "", -1)
+	result, _ := strconv.ParseUint(cleaned, 16, 64)
+	return uint64(result)
+}
+
 func prometheusHandler(w http.ResponseWriter, r *http.Request) {
-	metrics := []string{}
+	promValues = []string{}
+	prefix = utils.Getenv("PREFIX", "").(string)
+	if prefix != "" {
+		prefix = prefix + "_"
+	}
+
+	secondsOnline := time.Now().Sub(utils.StartTime).Seconds()
 	allFails := failures.All()
-	system := fmt.Sprintf("statping_total_failures %v\n", allFails)
+
-	system += fmt.Sprintf("statping_total_services %v", len(services.All()))
+	var m runtime.MemStats
-	metrics = append(metrics, system)
+	runtime.ReadMemStats(&m)
+
+	httpMetrics := utils.GetHttpMetrics()
+
+	promValues = append(promValues, "# Statping Prometheus Exporter")
+
+	PrometheusComment("Statping Totals")
+	PrometheusKeyValue("total_failures", len(allFails))
+	PrometheusKeyValue("total_services", len(services.Services()))
+	PrometheusKeyValue("seconds_online", secondsOnline)
+
+	if secondsOnline < 5 {
+		return
+	}
+
 	for _, ser := range services.All() {
 		online := 1
 		if !ser.Online {
 			online = 0
 		}
-		met := fmt.Sprintf("statping_service_failures{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, ser.AllFailures().Count())
+		id := ser.Id
-		met += fmt.Sprintf("statping_service_latency{id=\"%v\" name=\"%v\"} %0.0f\n", ser.Id, ser.Name, (ser.Latency * 100))
+		name := ser.Name
-		met += fmt.Sprintf("statping_service_online{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, online)
+
-		met += fmt.Sprintf("statping_service_status_code{id=\"%v\" name=\"%v\"} %v\n", ser.Id, ser.Name, ser.LastStatusCode)
+		PrometheusComment(fmt.Sprintf("Service #%d '%s':", ser.Id, ser.Name))
-		met += fmt.Sprintf("statping_service_response_length{id=\"%v\" name=\"%v\"} %v", ser.Id, ser.Name, len([]byte(ser.LastResponse)))
+		PrometheusExportKey("service_failures", id, name, ser.AllFailures().Count())
-		metrics = append(metrics, met)
+		PrometheusExportKey("service_latency", id, name, ser.Latency*100)
+		PrometheusExportKey("service_online", id, name, online)
+		PrometheusExportKey("service_status_code", id, name, ser.LastStatusCode)
+		PrometheusExportKey("service_response_length", id, name, len([]byte(ser.LastResponse)))
+		PrometheusExportKey("service_ping_time", id, name, ser.PingTime)
+		PrometheusExportKey("service_last_latency", id, name, ser.LastLatency)
+		PrometheusExportKey("service_last_lookup", id, name, ser.LastLookupTime)
+		PrometheusExportKey("service_last_check", id, name, time.Now().Sub(ser.LastCheck).Milliseconds())
+		//PrometheusExportKey("service_online_seconds", id, name, ser.SecondsOnline)
+		//PrometheusExportKey("service_offline_seconds", id, name, ser.SecondsOffline)
+
 	}
-	output := strings.Join(metrics, "\n")
+
+	for _, notif := range notifications.All() {
+		PrometheusComment(fmt.Sprintf("Notifier %s:", notif.Method))
+		PrometheusExportKey("notifier_on_success", notif.Id, notif.Method, 0)
+		PrometheusExportKey("notifier_on_failure", notif.Id, notif.Method, 0)
+	}
+
+	PrometheusComment("HTTP Metrics")
+	PrometheusKeyValue("http_errors", httpMetrics.Errors)
+	PrometheusKeyValue("http_requests", httpMetrics.Requests)
+	PrometheusKeyValue("http_bytes", httpMetrics.Bytes)
+	PrometheusKeyValue("http_request_milliseconds", httpMetrics.Milliseconds)
+
+	// https://golang.org/pkg/runtime/#MemStats
+	PrometheusComment("Golang Metrics")
+	PrometheusKeyValue("go_heap_allocated", m.Alloc)
+	PrometheusKeyValue("go_total_allocated", m.TotalAlloc)
+	PrometheusKeyValue("go_heap_in_use", m.HeapInuse)
+	PrometheusKeyValue("go_heap_objects", m.HeapObjects)
+	PrometheusKeyValue("go_heap_idle", m.HeapIdle)
+	PrometheusKeyValue("go_heap_released", m.HeapReleased)
+	PrometheusKeyValue("go_heap_frees", m.Frees)
+	PrometheusKeyValue("go_lookups", m.Lookups)
+	PrometheusKeyValue("go_system", m.Sys)
+	PrometheusKeyValue("go_number_gc", m.NumGC)
+	PrometheusKeyValue("go_number_gc_forced", m.NumForcedGC)
+	PrometheusKeyValue("go_goroutines", runtime.NumGoroutine())
+
+	output := strings.Join(promValues, "\n")
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(output))
 }
+
+func PrometheusKeyValue(keyName string, value interface{}) {
+	val := promValue(value)
+	prom := fmt.Sprintf("%sstatping_%s %s", prefix, keyName, val)
+	promValues = append(promValues, prom)
+}
+
+func PrometheusExportKey(keyName string, id int64, name string, value interface{}) {
+	val := promValue(value)
+	prom := fmt.Sprintf("%sstatping_%s{id=\"%d\" name=\"%s\"} %s", prefix, keyName, id, name, val)
+	promValues = append(promValues, prom)
+}
+
+func PrometheusComment(comment string) {
+	prom := fmt.Sprintf("\n# %v", comment)
+	promValues = append(promValues, prom)
+}
+
+func promValue(val interface{}) string {
+	var newVal string
+	switch v := val.(type) {
+	case float64:
+		newVal = fmt.Sprintf("%.4f", v)
+	default:
+		newVal = fmt.Sprintf("%v", v)
+	}
+	return newVal
+}

types/notifications/struct.go

@@ -63,9 +63,15 @@ type Notification struct {
 	Running     chan bool          `gorm:"-" json:"-"`
 	testable    bool               `gorm:"-" json:"testable"`
 
+	Hits notificationHits
 	Notifier
 }
 
+type notificationHits struct {
+	onSuccess int64 `gorm:"-" json:"-"`
+	onFailure int64 `gorm:"-" json:"-"`
+}
+
 // QueueData is the struct for the messaging queue with service
 type QueueData struct {
 	Id   string

types/services/database.go

@@ -21,12 +21,16 @@ func Find(id int64) (*Service, error) {
 	return srv, nil
 }
 
-func All() []*Service {
+func all() []*Service {
 	var services []*Service
 	DB().Find(&services)
 	return services
 }
 
+func All() map[int64]*Service {
+	return allServices
+}
+
 func (s *Service) Create() error {
 	err := DB().Create(&s)
 	if err.Error() != nil {

types/services/methods.go

@@ -66,7 +66,7 @@ func SelectAllServices(start bool) (map[int64]*Service, error) {
 		return allServices, nil
 	}
 
-	for _, s := range All() {
+	for _, s := range all() {
 
 		allServices[s.Id] = s
 

types/services/routine.go

@@ -87,6 +87,8 @@ func isIPv6(address string) bool {
 
 // checkIcmp will send a ICMP ping packet to the service
 func CheckIcmp(s *Service, record bool) *Service {
+	defer s.updateLastCheck()
+
 	p := fastping.NewPinger()
 	resolveIP := "ip4:icmp"
 	if isIPv6(s.Domain) {
@@ -113,6 +115,8 @@ func CheckIcmp(s *Service, record bool) *Service {
 
 // checkTcp will check a TCP service
 func CheckTcp(s *Service, record bool) *Service {
+	defer s.updateLastCheck()
+
 	dnsLookup, err := dnsCheck(s)
 	if err != nil {
 		if record {
@@ -151,8 +155,14 @@ func CheckTcp(s *Service, record bool) *Service {
 	return s
 }
 
+func (s *Service) updateLastCheck() {
+	s.LastCheck = time.Now()
+}
+
 // checkHttp will check a HTTP service
 func CheckHttp(s *Service, record bool) *Service {
+	defer s.updateLastCheck()
+
 	dnsLookup, err := dnsCheck(s)
 	if err != nil {
 		if record {
@@ -229,12 +239,16 @@ func recordSuccess(s *Service) {
 	}
 	log.WithFields(utils.ToFields(hit, s)).Infoln(
 		fmt.Sprintf("Service #%d '%v' Successful Response: %0.2f ms | Lookup in: %0.2f ms | Online: %v | Interval: %d seconds", s.Id, s.Name, hit.Latency*1000, hit.PingTime*1000, s.Online, s.Interval))
+	s.LastLookupTime = int64(hit.PingTime * 1000)
+	s.LastLatency = int64(hit.Latency * 1000)
 	//notifier.OnSuccess(s)
 	s.SuccessNotified = true
 }
 
 // recordFailure will create a new 'Failure' record in the database for a offline service
 func recordFailure(s *Service, issue string) {
+	s.LastOffline = time.Now().UTC()
+
 	fail := &failures.Failure{
 		Service:   s.Id,
 		Issue:     issue,

types/services/struct.go

@@ -58,9 +58,16 @@ type Service struct {
 	SuccessNotified     bool                `gorm:"-" json:"-"`                                                                          // Is 'true' if the user has already be informed that the Services now again available
 	LastStatusCode      int                 `gorm:"-" json:"status_code"`
 	LastOnline          time.Time           `gorm:"-" json:"last_success"`
+	LastOffline         time.Time           `gorm:"-" json:"last_error"`
 	Failures            []*failures.Failure `gorm:"-" json:"failures,omitempty" scope:"user,admin"`
 	AllCheckins         []*checkins.Checkin `gorm:"-" json:"checkins,omitempty" scope:"user,admin"`
 	Stats               *Stats              `gorm:"-" json:"stats,omitempty"`
+	LastLookupTime      int64               `gorm:"-" json:"-"`
+	LastLatency         int64               `gorm:"-" json:"-"`
+	LastCheck           time.Time           `gorm:"-" json:"-"`
+
+	SecondsOnline  int64 `gorm:"-" json:"-"`
+	SecondsOffline int64 `gorm:"-" json:"-"`
 }
 
 type Stats struct {

utils/metrics.go

@@ -0,0 +1,29 @@
+package utils
+
+import "time"
+
+func init() {
+	httpMetric = new(Metrics)
+}
+
+var (
+	httpMetric *Metrics
+	StartTime  = time.Now()
+)
+
+type Metrics struct {
+	Requests     int64
+	Errors       int64
+	Bytes        int64
+	Milliseconds int64
+	OnlineTime   time.Time
+}
+
+func (h *Metrics) Reset() {
+	httpMetric = new(Metrics)
+}
+
+func GetHttpMetrics() *Metrics {
+	defer httpMetric.Reset()
+	return httpMetric
+}

utils/utils.go

@@ -374,7 +374,9 @@ func SaveFile(filename string, data []byte) error {
 func HttpRequest(url, method string, content interface{}, headers []string, body io.Reader, timeout time.Duration, verifySSL bool) ([]byte, *http.Response, error) {
 	var err error
 	var req *http.Request
+	t1 := time.Now()
 	if req, err = http.NewRequest(method, url, body); err != nil {
+		httpMetric.Errors++
 		return nil, nil, err
 	}
 	req.Header.Set("User-Agent", "Statping")
@@ -424,10 +426,18 @@ func HttpRequest(url, method string, content interface{}, headers []string, body
 	}
 
 	if resp, err = client.Do(req); err != nil {
+		httpMetric.Errors++
 		return nil, resp, err
 	}
 	defer resp.Body.Close()
 	contents, err := ioutil.ReadAll(resp.Body)
+
+	// record HTTP metrics
+	t2 := time.Now().Sub(t1).Milliseconds()
+	httpMetric.Requests++
+	httpMetric.Milliseconds += t2 / httpMetric.Requests
+	httpMetric.Bytes += int64(len(contents))
+
 	return contents, resp, err
 }