Merge pull request #668 from statping/metrics-update

prometheus metric updates

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.90.53 (06-16-2020)
+- Modified most of the key's for prometheus metrics
+- Added Database Stats in prometheus metrics
+- Added object query counts in prometheus metrics
+
 # 0.90.52 (06-15-2020)
 - Fixed NOT NULL sql field
 

cmd/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/statping/statping/source"
 	"github.com/statping/statping/types/configs"
 	"github.com/statping/statping/types/core"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/types/services"
 	"github.com/statping/statping/utils"
 	"os"
@@ -174,6 +175,8 @@ func InitApp() error {
 	if _, err := core.Select(); err != nil {
 		return err
 	}
+	// init prometheus metrics
+	metrics.InitMetrics()
 	// select all services in database and store services in a mapping of Service pointers
 	if _, err := services.SelectAllServices(true); err != nil {
 		return err

database/database.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 	"github.com/jinzhu/gorm"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 	"strings"
 	"time"
@@ -119,6 +120,17 @@ func (it *Db) ChunkSize() int {
 	}
 }
 
+func Routine() {
+	for {
+		if database.DB() == nil {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		metrics.CollectDatabase(database.DB().Stats())
+		time.Sleep(5 * time.Second)
+	}
+}
+
 func (it *Db) GormDB() *gorm.DB {
 	return it.Database
 }
@@ -184,6 +196,7 @@ func Openw(dialect string, args ...interface{}) (db Database, err error) {
 		return nil, err
 	}
 	database = Wrap(gormdb)
+	go Routine()
 	return database, err
 }
 

handlers/api_test.go

@@ -248,9 +248,9 @@ func TestMainApiRoutes(t *testing.T) {
 			ExpectedContains: []string{
 				`go_goroutines`,
 				`go_memstats_alloc_bytes`,
-				`http_duration_seconds_count`,
+				`process_cpu_seconds_total`,
-				`http_response_bytes_count`,
+				`promhttp_metric_handler_requests_total`,
-				`service_success`,
+				`go_threads`,
 			},
 		},
 	}

handlers/middleware.go

@@ -7,9 +7,9 @@ import (
 	"fmt"
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/statping/statping/types/core"
 	"github.com/statping/statping/types/errors"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 	"io"
 	"net/http"
@@ -19,12 +19,8 @@ import (
 )
 
 var (
-	authUser     string
+	authUser string
-	authPass     string
+	authPass string
-	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "http_duration_seconds",
-		Help: "Duration of HTTP requests.",
-	}, []string{"path"})
 )
 
 // Gzip Compression
@@ -178,7 +174,7 @@ func prometheusMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		route := mux.CurrentRoute(r)
 		path, _ := route.GetPathTemplate()
-		timer := prometheus.NewTimer(httpDuration.WithLabelValues(path))
+		timer := prometheus.NewTimer(metrics.Timer(path))
 		next.ServeHTTP(w, r)
 		timer.ObserveDuration()
 	})

handlers/scope.go

@@ -26,7 +26,7 @@ type scope struct {
 	scope string
 }
 
-// MarshalJSON for Scopr
+// TODO: make a better way to parse
 func (s scope) MarshalJSON() ([]byte, error) {
 	svc := reflect.ValueOf(s.data)
 	if svc.Kind() == reflect.Slice {
@@ -40,6 +40,7 @@ func (s scope) MarshalJSON() ([]byte, error) {
 	return json.Marshal(SafeJson(svc, s.scope))
 }
 
+// TODO: make a better way to parse
 func SafeJson(val reflect.Value, scope string) map[string]interface{} {
 	thisData := make(map[string]interface{})
 	if val.Kind() == reflect.Interface && !val.IsNil() {
@@ -89,6 +90,7 @@ func SafeJson(val reflect.Value, scope string) map[string]interface{} {
 	return thisData
 }
 
+// TODO: make a better way to parse
 func forTag(tags []string, scope string) bool {
 	for _, v := range tags {
 		if v == scope {

types/checkins/database.go

@@ -2,6 +2,7 @@ package checkins
 
 import (
 	"github.com/statping/statping/database"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 )
 
@@ -13,6 +14,10 @@ func SetDB(database database.Database) {
 	dbHits = database.Model(&CheckinHit{})
 }
 
+func (c *Checkin) AfterFind() {
+	metrics.Query("checkin", "find")
+}
+
 func Find(id int64) (*Checkin, error) {
 	var checkin Checkin
 	q := db.Where("id = ?", id).Find(&checkin)

types/core/database.go

@@ -3,6 +3,7 @@ package core
 import (
 	"github.com/pkg/errors"
 	"github.com/statping/statping/database"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/types/null"
 	"github.com/statping/statping/utils"
 )
@@ -13,6 +14,10 @@ func SetDB(database database.Database) {
 	db = database.Model(&Core{})
 }
 
+func (c *Core) AfterFind() {
+	metrics.Query("core", "find")
+}
+
 func Select() (*Core, error) {
 	var c Core
 	// SelectCore will return the CoreApp global variable and the settings/configs for Statping

types/failures/database.go

@@ -15,14 +15,27 @@ func DB() database.Database {
 	return db
 }
 
-func All() []*Failure {
+func (f *Failure) AfterFind() {
-	var failures []*Failure
+	metrics.Query("failure", "find")
-	db.Find(&failures)
+}
-	return failures
+
+func (f *Failure) AfterUpdate() {
+	metrics.Query("failure", "update")
+}
+
+func (f *Failure) AfterDelete() {
+	metrics.Query("failure", "delete")
 }
 
 func (f *Failure) AfterCreate() {
 	metrics.Inc("failure", f.Service)
+	metrics.Query("failure", "create")
+}
+
+func All() []*Failure {
+	var failures []*Failure
+	db.Find(&failures)
+	return failures
 }
 
 func (f *Failure) Create() error {

types/groups/database.go

@@ -3,6 +3,7 @@ package groups
 import (
 	"github.com/statping/statping/database"
 	"github.com/statping/statping/types/errors"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 	"sort"
 )
@@ -16,6 +17,22 @@ func SetDB(database database.Database) {
 	db = database.Model(&Group{})
 }
 
+func (g *Group) AfterFind() {
+	metrics.Query("group", "find")
+}
+
+func (g *Group) AfterUpdate() {
+	metrics.Query("group", "update")
+}
+
+func (g *Group) AfterDelete() {
+	metrics.Query("group", "delete")
+}
+
+func (g *Group) AfterCreate() {
+	metrics.Query("group", "create")
+}
+
 func Find(id int64) (*Group, error) {
 	var group Group
 	q := db.Where("id = ?", id).Find(&group)

types/hits/database.go

@@ -14,8 +14,21 @@ func SetDB(database database.Database) {
 	db = database.Model(&Hit{})
 }
 
+func (h *Hit) AfterFind() {
+	metrics.Query("hit", "find")
+}
+
+func (h *Hit) AfterUpdate() {
+	metrics.Query("hit", "update")
+}
+
+func (h *Hit) AfterDelete() {
+	metrics.Query("hit", "delete")
+}
+
 func (h *Hit) AfterCreate() {
 	metrics.Inc("success", h.Service)
+	metrics.Query("hit", "create")
 }
 
 func (h *Hit) Create() error {

types/incidents/database.go

@@ -2,6 +2,7 @@ package incidents
 
 import (
 	"github.com/statping/statping/database"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 )
 
@@ -16,6 +17,38 @@ func SetDB(database database.Database) {
 	dbUpdate = database.Model(&IncidentUpdate{})
 }
 
+func (i *Incident) AfterFind() {
+	metrics.Query("incident", "find")
+}
+
+func (i *Incident) AfterCreate() {
+	metrics.Query("incident", "create")
+}
+
+func (i *Incident) AfterUpdate() {
+	metrics.Query("incident", "update")
+}
+
+func (i *Incident) AfterDelete() {
+	metrics.Query("incident", "delete")
+}
+
+func (i *IncidentUpdate) AfterFind() {
+	metrics.Query("incident_update", "find")
+}
+
+func (i *IncidentUpdate) AfterCreate() {
+	metrics.Query("incident_update", "create")
+}
+
+func (i *IncidentUpdate) AfterUpdate() {
+	metrics.Query("incident_update", "update")
+}
+
+func (i *IncidentUpdate) AfterDelete() {
+	metrics.Query("incident_update", "delete")
+}
+
 func FindUpdate(uid int64) (*IncidentUpdate, error) {
 	var update IncidentUpdate
 	q := dbUpdate.Where("id = ?", uid).Find(&update)

types/messages/hooks.go

@@ -1,6 +1,7 @@
 package messages
 
 import (
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 )
 
@@ -12,3 +13,19 @@ func (m *Message) BeforeCreate() (err error) {
 	}
 	return
 }
+
+func (m *Message) AfterFind() {
+	metrics.Query("message", "find")
+}
+
+func (m *Message) AfterCreate() {
+	metrics.Query("message", "create")
+}
+
+func (m *Message) AfterUpdate() {
+	metrics.Query("message", "update")
+}
+
+func (m *Message) AfterDelete() {
+	metrics.Query("message", "delete")
+}

types/metrics/database.go

@@ -0,0 +1,40 @@
+package metrics
+
+import (
+	"database/sql"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	// service is online if set to 1, offline if 0
+	databaseStats = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "statping",
+			Name:      "database",
+			Help:      "If service is online",
+		}, []string{"metric"},
+	)
+
+	queryStats = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "statping",
+			Name:      "query",
+			Help:      "If service is online",
+		}, []string{"type", "method"},
+	)
+)
+
+func Query(objType, method string) {
+	queryStats.WithLabelValues(objType, method).Inc()
+}
+
+func CollectDatabase(stats sql.DBStats) {
+	databaseStats.WithLabelValues("max_open_connections").Set(float64(stats.MaxOpenConnections))
+	databaseStats.WithLabelValues("open_connections").Set(float64(stats.OpenConnections))
+	databaseStats.WithLabelValues("in_use_connections").Set(float64(stats.InUse))
+	databaseStats.WithLabelValues("idle_connections").Set(float64(stats.Idle))
+	databaseStats.WithLabelValues("wait_count").Set(float64(stats.WaitCount))
+	databaseStats.WithLabelValues("wait_duration_seconds").Set(stats.WaitDuration.Seconds())
+	databaseStats.WithLabelValues("idle_connections_closed").Set(float64(stats.MaxIdleClosed))
+	databaseStats.WithLabelValues("lifetime_connections_closed").Set(float64(stats.MaxLifetimeClosed))
+}

types/metrics/metrics.go

@@ -8,37 +8,47 @@ import (
 var (
 	utilsHttpRequestDur = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: "http_requests_duration",
+			Namespace: "statping",
-			Help: "How many successful requests for a service",
+			Name:      "http_requests_duration",
+			Help:      "Duration for h",
 		},
 		[]string{"url", "method"},
 	)
 
 	utilsHttpRequestBytes = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: "http_response_bytes",
+			Namespace: "statping",
-			Help: "How many successful requests for a service",
+			Name:      "http_response_bytes",
+			Help:      "Response in bytes for a HTTP services",
 		},
 		[]string{"url", "method"},
 	)
+
+	httpDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "statping",
+			Name:      "http_duration_seconds",
+			Help:      "Duration of HTTP requests from the utils package",
+		}, []string{"path"})
 )
 
-func init() {
+func InitMetrics() {
 	prometheus.MustRegister(
 		serviceOnline,
 		serviceFailures,
 		serviceSuccess,
 		serviceStatusCode,
-		serviceLatencyDuration,
+		serviceDuration,
 		utilsHttpRequestDur,
 		utilsHttpRequestBytes,
+		httpDuration,
+		databaseStats,
+		queryStats,
 	)
 }
 
 func Histo(method string, value float64, labels ...interface{}) {
 	switch method {
-	case "latency":
-		serviceLatencyDuration.WithLabelValues(convert(labels)...).Observe(value)
 	case "duration":
 		utilsHttpRequestDur.WithLabelValues(convert(labels)...).Observe(value)
 	case "bytes":
@@ -46,9 +56,17 @@ func Histo(method string, value float64, labels ...interface{}) {
 	}
 }
 
+func Timer(labels ...interface{}) prometheus.Observer {
+	return httpDuration.WithLabelValues(convert(labels)...)
+}
+
+func ServiceTimer(labels ...interface{}) prometheus.Observer {
+	return serviceDuration.WithLabelValues(convert(labels)...)
+}
+
 func Gauge(method string, value float64, labels ...interface{}) {
 	switch method {
-	case "service":
+	case "status_code":
 		serviceStatusCode.WithLabelValues(convert(labels)...).Set(value)
 	case "online":
 		serviceOnline.WithLabelValues(convert(labels)...).Set(value)

types/metrics/services.go

@@ -6,17 +6,19 @@ var (
 	// service is online if set to 1, offline if 0
 	serviceOnline = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "service_online",
+			Namespace: "statping",
-			Help: "If service is online",
+			Name:      "service_online",
+			Help:      "If service is online",
 		},
-		[]string{"service"},
+		[]string{"service", "name", "type"},
 	)
 
 	// service failures
 	serviceFailures = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "service_failures",
+			Namespace: "statping",
-			Help: "How many failures occur for a service",
+			Name:      "service_failures",
+			Help:      "How many failures occur for a service",
 		},
 		[]string{"service"},
 	)
@@ -24,17 +26,19 @@ var (
 	// successful hits for a service
 	serviceSuccess = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "service_success",
+			Namespace: "statping",
-			Help: "How many successful requests for a service",
+			Name:      "service_success",
+			Help:      "How many successful requests for a service",
 		},
 		[]string{"service"},
 	)
 
 	// service check latency
-	serviceLatencyDuration = prometheus.NewHistogramVec(
+	serviceDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name: "service_latency",
+			Namespace: "statping",
-			Help: "How many successful requests for a service",
+			Name:      "service_duration",
+			Help:      "Service request duration for a success response",
 		},
 		[]string{"service"},
 	)
@@ -42,8 +46,9 @@ var (
 	// http status code for a service
 	serviceStatusCode = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "service_status_code",
+			Namespace: "statping",
-			Help: "HTTP Status code for a service",
+			Name:      "service_status_code",
+			Help:      "HTTP Status code for a service",
 		},
 		[]string{"service"},
 	)

types/notifications/hooks.go

@@ -1,10 +1,26 @@
 package notifications
 
-import "github.com/statping/statping/utils"
+import (
+	"github.com/statping/statping/types/metrics"
+	"github.com/statping/statping/utils"
+)
 
 // AfterFind for Notification will set the timezone
 func (n *Notification) AfterFind() (err error) {
 	n.CreatedAt = utils.Now()
 	n.UpdatedAt = utils.Now()
+	metrics.Query("notifier", "find")
 	return
 }
+
+func (n *Notification) AfterCreate() {
+	metrics.Query("notifier", "create")
+}
+
+func (n *Notification) AfterUpdate() {
+	metrics.Query("notifier", "update")
+}
+
+func (n *Notification) AfterDelete() {
+	metrics.Query("notifier", "delete")
+}

types/services/database.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/statping/statping/database"
 	"github.com/statping/statping/types/errors"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 	"sort"
 )
@@ -14,6 +15,18 @@ var (
 	allServices map[int64]*Service
 )
 
+func (s *Service) AfterFind() {
+	metrics.Query("service", "find")
+}
+
+func (s *Service) AfterUpdate() {
+	metrics.Query("service", "update")
+}
+
+func (s *Service) AfterDelete() {
+	metrics.Query("service", "delete")
+}
+
 func init() {
 	allServices = make(map[int64]*Service)
 }
@@ -65,6 +78,7 @@ func (s *Service) Create() error {
 
 func (s *Service) AfterCreate() error {
 	allServices[s.Id] = s
+	metrics.Query("service", "create")
 	return nil
 }
 

types/services/routine.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/statping/statping/types/metrics"
 	"google.golang.org/grpc"
 	"net"
@@ -87,6 +88,8 @@ func isIPv6(address string) bool {
 // checkIcmp will send a ICMP ping packet to the service
 func CheckIcmp(s *Service, record bool) (*Service, error) {
 	defer s.updateLastCheck()
+	timer := prometheus.NewTimer(metrics.ServiceTimer(s.Id))
+	defer timer.ObserveDuration()
 
 	if err := utils.Ping(s.Domain, s.Timeout); err != nil {
 		if record {
@@ -102,6 +105,8 @@ func CheckIcmp(s *Service, record bool) (*Service, error) {
 // CheckGrpc will check a gRPC service
 func CheckGrpc(s *Service, record bool) (*Service, error) {
 	defer s.updateLastCheck()
+	timer := prometheus.NewTimer(metrics.ServiceTimer(s.Id))
+	defer timer.ObserveDuration()
 
 	dnsLookup, err := dnsCheck(s)
 	if err != nil {
@@ -147,6 +152,8 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
 // checkTcp will check a TCP service
 func CheckTcp(s *Service, record bool) (*Service, error) {
 	defer s.updateLastCheck()
+	timer := prometheus.NewTimer(metrics.ServiceTimer(s.Id))
+	defer timer.ObserveDuration()
 
 	dnsLookup, err := dnsCheck(s)
 	if err != nil {
@@ -212,6 +219,8 @@ func (s *Service) updateLastCheck() {
 // checkHttp will check a HTTP service
 func CheckHttp(s *Service, record bool) (*Service, error) {
 	defer s.updateLastCheck()
+	timer := prometheus.NewTimer(metrics.ServiceTimer(s.Id))
+	defer timer.ObserveDuration()
 
 	dnsLookup, err := dnsCheck(s)
 	if err != nil {
@@ -273,10 +282,10 @@ func CheckHttp(s *Service, record bool) (*Service, error) {
 		return s, err
 	}
 	s.Latency = utils.Now().Sub(t1).Microseconds()
-	metrics.Histo("latency", utils.Now().Sub(t1).Seconds(), s.Id)
 	s.LastResponse = string(content)
 	s.LastStatusCode = res.StatusCode
-	metrics.Gauge("service", float64(res.StatusCode), s.Id)
+
+	metrics.Gauge("status_code", float64(res.StatusCode), s.Id)
 
 	if s.Expected.String != "" {
 		match, err := regexp.MatchString(s.Expected.String, string(content))
@@ -320,9 +329,9 @@ func recordSuccess(s *Service) {
 		fmt.Sprintf("Service #%d '%v' Successful Response: %s | Lookup in: %s | Online: %v | Interval: %d seconds", s.Id, s.Name, humanMicro(hit.Latency), humanMicro(hit.PingTime), s.Online, s.Interval))
 	s.LastLookupTime = hit.PingTime
 	s.LastLatency = hit.Latency
+	metrics.Gauge("online", 1., s.Id, s.Name, s.Type)
 	sendSuccess(s)
 	s.SuccessNotified = true
-	metrics.Gauge("online", 1., s.Id)
 }
 
 func AddNotifier(n ServiceNotifier) {
@@ -374,8 +383,8 @@ func recordFailure(s *Service, issue string) {
 	s.Online = false
 	s.SuccessNotified = false
 	s.DownText = s.DowntimeText()
+	metrics.Gauge("online", 0., s.Id, s.Name, s.Type)
 	sendFailure(s, fail)
-	metrics.Gauge("online", 0., s.Id)
 }
 
 func sendFailure(s *Service, f *failures.Failure) {

types/users/database.go

@@ -2,6 +2,7 @@ package users
 
 import (
 	"github.com/statping/statping/database"
+	"github.com/statping/statping/types/metrics"
 	"github.com/statping/statping/utils"
 )
 
@@ -14,6 +15,22 @@ func SetDB(database database.Database) {
 	db = database.Model(&User{})
 }
 
+func (u *User) AfterFind() {
+	metrics.Query("user", "find")
+}
+
+func (u *User) AfterCreate() {
+	metrics.Query("user", "create")
+}
+
+func (u *User) AfterUpdate() {
+	metrics.Query("user", "update")
+}
+
+func (u *User) AfterDelete() {
+	metrics.Query("user", "delete")
+}
+
 func Find(id int64) (*User, error) {
 	var user User
 	q := db.Where("id = ?", id).Find(&user)

version.txt

@@ -1 +1 @@
-0.90.52
+0.90.53