2020-03-04 10:29:00 +00:00
|
|
|
package services
|
|
|
|
|
|
|
|
import (
|
|
|
|
"crypto/sha1"
|
2020-05-20 06:41:50 +00:00
|
|
|
"crypto/tls"
|
|
|
|
"crypto/x509"
|
2020-03-04 10:29:00 +00:00
|
|
|
"encoding/hex"
|
|
|
|
"fmt"
|
2020-03-10 05:24:35 +00:00
|
|
|
"github.com/statping/statping/types"
|
2020-05-20 06:41:50 +00:00
|
|
|
"github.com/statping/statping/types/errors"
|
2020-04-14 16:08:50 +00:00
|
|
|
"github.com/statping/statping/types/failures"
|
|
|
|
"github.com/statping/statping/types/hits"
|
2020-03-09 18:17:55 +00:00
|
|
|
"github.com/statping/statping/utils"
|
2020-05-20 06:41:50 +00:00
|
|
|
"io/ioutil"
|
2020-04-14 16:08:50 +00:00
|
|
|
"sort"
|
2020-03-04 10:29:00 +00:00
|
|
|
"strconv"
|
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
|
|
|
const limitedFailures = 25
|
|
|
|
|
2020-05-20 06:41:50 +00:00
|
|
|
func (s *Service) LoadTLSCert() (*tls.Config, error) {
|
2020-05-21 07:10:35 +00:00
|
|
|
if s.TLSCert.String == "" || s.TLSCertKey.String == "" {
|
2020-05-20 06:41:50 +00:00
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// load TLS cert and key from file path or PEM format
|
|
|
|
var cert tls.Certificate
|
|
|
|
var err error
|
|
|
|
tlsCertExtension := utils.FileExtension(s.TLSCert.String)
|
|
|
|
tlsCertKeyExtension := utils.FileExtension(s.TLSCertKey.String)
|
|
|
|
if tlsCertExtension == "" && tlsCertKeyExtension == "" {
|
|
|
|
cert, err = tls.X509KeyPair([]byte(s.TLSCert.String), []byte(s.TLSCertKey.String))
|
|
|
|
} else {
|
|
|
|
cert, err = tls.LoadX509KeyPair(s.TLSCert.String, s.TLSCertKey.String)
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.Wrap(err, "issue loading X509KeyPair")
|
|
|
|
}
|
|
|
|
|
2020-05-21 07:10:35 +00:00
|
|
|
config := &tls.Config{
|
|
|
|
Certificates: []tls.Certificate{cert},
|
|
|
|
InsecureSkipVerify: s.TLSCertRoot.String == "",
|
|
|
|
}
|
|
|
|
|
|
|
|
if s.TLSCertRoot.String == "" {
|
|
|
|
return config, nil
|
2020-05-20 06:41:50 +00:00
|
|
|
}
|
2020-05-21 07:10:35 +00:00
|
|
|
|
|
|
|
// create Root CA pool or use Root CA provided
|
|
|
|
rootCA := s.TLSCertRoot.String
|
|
|
|
caCert, err := ioutil.ReadFile(rootCA)
|
2020-05-20 06:41:50 +00:00
|
|
|
if err != nil {
|
2020-05-21 07:10:35 +00:00
|
|
|
return nil, errors.Wrap(err, "issue reading root CA file: "+rootCA)
|
2020-05-20 06:41:50 +00:00
|
|
|
}
|
|
|
|
caCertPool := x509.NewCertPool()
|
|
|
|
caCertPool.AppendCertsFromPEM(caCert)
|
|
|
|
|
2020-05-21 07:10:35 +00:00
|
|
|
config.RootCAs = caCertPool
|
|
|
|
|
|
|
|
return config, nil
|
2020-05-20 06:41:50 +00:00
|
|
|
}
|
|
|
|
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) Duration() time.Duration {
|
2020-03-04 10:29:00 +00:00
|
|
|
return time.Duration(s.Interval) * time.Second
|
|
|
|
}
|
|
|
|
|
2020-04-14 16:08:50 +00:00
|
|
|
// Start will create a channel for the service checking go routine
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) UptimeData(hits []*hits.Hit, fails []*failures.Failure) (*UptimeSeries, error) {
|
2020-04-14 16:08:50 +00:00
|
|
|
if len(hits) == 0 {
|
|
|
|
return nil, errors.New("service does not have any successful hits")
|
|
|
|
}
|
|
|
|
// if theres no failures, then its been online 100%,
|
|
|
|
// return a series from created time, to current.
|
|
|
|
if len(fails) == 0 {
|
|
|
|
fistHit := hits[0]
|
|
|
|
duration := utils.Now().Sub(fistHit.CreatedAt).Milliseconds()
|
|
|
|
set := []series{
|
|
|
|
{
|
|
|
|
Start: fistHit.CreatedAt,
|
|
|
|
End: utils.Now(),
|
|
|
|
Duration: duration,
|
|
|
|
Online: true,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
out := &UptimeSeries{
|
|
|
|
Start: fistHit.CreatedAt,
|
|
|
|
End: utils.Now(),
|
|
|
|
Uptime: duration,
|
|
|
|
Downtime: 0,
|
|
|
|
Series: set,
|
|
|
|
}
|
|
|
|
return out, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
tMap := make(map[time.Time]bool)
|
|
|
|
|
|
|
|
for _, v := range hits {
|
|
|
|
tMap[v.CreatedAt] = true
|
|
|
|
}
|
|
|
|
for _, v := range fails {
|
|
|
|
tMap[v.CreatedAt] = false
|
|
|
|
}
|
|
|
|
|
|
|
|
var servs []ser
|
|
|
|
for t, v := range tMap {
|
|
|
|
s := ser{
|
|
|
|
Time: t,
|
|
|
|
Online: v,
|
|
|
|
}
|
|
|
|
servs = append(servs, s)
|
|
|
|
}
|
|
|
|
if len(servs) == 0 {
|
|
|
|
return nil, errors.New("error generating uptime data structure")
|
|
|
|
}
|
|
|
|
sort.Sort(ByTime(servs))
|
|
|
|
|
|
|
|
var allTimes []series
|
|
|
|
online := servs[0].Online
|
|
|
|
thisTime := servs[0].Time
|
|
|
|
for i := 0; i < len(servs); i++ {
|
|
|
|
v := servs[i]
|
|
|
|
if v.Online != online {
|
|
|
|
s := series{
|
|
|
|
Start: thisTime,
|
|
|
|
End: v.Time,
|
|
|
|
Duration: v.Time.Sub(thisTime).Milliseconds(),
|
|
|
|
Online: online,
|
|
|
|
}
|
|
|
|
allTimes = append(allTimes, s)
|
|
|
|
thisTime = v.Time
|
|
|
|
online = v.Online
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(allTimes) == 0 {
|
|
|
|
return nil, errors.New("error generating uptime series structure")
|
|
|
|
}
|
|
|
|
|
|
|
|
first := servs[0].Time
|
|
|
|
last := servs[len(servs)-1].Time
|
|
|
|
if !s.Online {
|
|
|
|
s := series{
|
|
|
|
Start: allTimes[len(allTimes)-1].End,
|
|
|
|
End: utils.Now(),
|
|
|
|
Duration: utils.Now().Sub(last).Milliseconds(),
|
|
|
|
Online: s.Online,
|
|
|
|
}
|
|
|
|
allTimes = append(allTimes, s)
|
|
|
|
} else {
|
|
|
|
l := allTimes[len(allTimes)-1]
|
|
|
|
s := series{
|
|
|
|
Start: l.Start,
|
|
|
|
End: utils.Now(),
|
|
|
|
Duration: utils.Now().Sub(l.Start).Milliseconds(),
|
|
|
|
Online: true,
|
|
|
|
}
|
|
|
|
allTimes = append(allTimes, s)
|
|
|
|
}
|
|
|
|
|
|
|
|
response := &UptimeSeries{
|
|
|
|
Start: first,
|
|
|
|
End: last,
|
|
|
|
Uptime: addDurations(allTimes, true),
|
|
|
|
Downtime: addDurations(allTimes, false),
|
|
|
|
Series: allTimes,
|
|
|
|
}
|
|
|
|
|
|
|
|
return response, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func addDurations(s []series, on bool) int64 {
|
|
|
|
var dur int64
|
|
|
|
for _, v := range s {
|
|
|
|
if v.Online == on {
|
|
|
|
dur += v.Duration
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dur
|
|
|
|
}
|
|
|
|
|
|
|
|
type ser struct {
|
|
|
|
Time time.Time
|
|
|
|
Online bool
|
|
|
|
}
|
|
|
|
|
|
|
|
type UptimeSeries struct {
|
|
|
|
Start time.Time `json:"start"`
|
|
|
|
End time.Time `json:"end"`
|
|
|
|
Uptime int64 `json:"uptime"`
|
|
|
|
Downtime int64 `json:"downtime"`
|
|
|
|
Series []series `json:"series"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type ByTime []ser
|
|
|
|
|
|
|
|
func (a ByTime) Len() int { return len(a) }
|
|
|
|
func (a ByTime) Less(i, j int) bool { return a[i].Time.Before(a[j].Time) }
|
|
|
|
func (a ByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
|
|
|
|
type series struct {
|
|
|
|
Start time.Time `json:"start"`
|
|
|
|
End time.Time `json:"end"`
|
|
|
|
Duration int64 `json:"duration"`
|
|
|
|
Online bool `json:"online"`
|
|
|
|
}
|
|
|
|
|
2020-03-04 10:29:00 +00:00
|
|
|
// Start will create a channel for the service checking go routine
|
|
|
|
func (s *Service) Start() {
|
|
|
|
if s.IsRunning() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
s.Running = make(chan bool)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Close will stop the go routine that is checking if service is online or not
|
|
|
|
func (s *Service) Close() {
|
|
|
|
if s.IsRunning() {
|
|
|
|
close(s.Running)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-10 05:24:35 +00:00
|
|
|
func humanMicro(val int64) string {
|
|
|
|
if val < 10000 {
|
|
|
|
return fmt.Sprintf("%d μs", val)
|
|
|
|
}
|
2020-03-13 04:06:06 +00:00
|
|
|
return fmt.Sprintf("%0.0f ms", float64(val)*0.001)
|
2020-03-10 05:24:35 +00:00
|
|
|
}
|
|
|
|
|
2020-03-04 10:29:00 +00:00
|
|
|
// IsRunning returns true if the service go routine is running
|
|
|
|
func (s *Service) IsRunning() bool {
|
|
|
|
if s.Running == nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-s.Running:
|
|
|
|
return false
|
|
|
|
default:
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-04 14:20:47 +00:00
|
|
|
func (s Service) Hash() string {
|
2020-03-04 10:29:00 +00:00
|
|
|
format := fmt.Sprintf("name:%sdomain:%sport:%dtype:%smethod:%s", s.Name, s.Domain, s.Port, s.Type, s.Method)
|
|
|
|
h := sha1.New()
|
|
|
|
h.Write([]byte(format))
|
|
|
|
return hex.EncodeToString(h.Sum(nil))
|
|
|
|
}
|
|
|
|
|
|
|
|
// SelectAllServices returns a slice of *core.Service to be store on []*core.Services
|
|
|
|
// should only be called once on startup.
|
|
|
|
func SelectAllServices(start bool) (map[int64]*Service, error) {
|
|
|
|
if len(allServices) > 0 {
|
|
|
|
return allServices, nil
|
|
|
|
}
|
2020-03-05 08:27:51 +00:00
|
|
|
for _, s := range all() {
|
2020-06-13 01:13:28 +00:00
|
|
|
|
2020-03-04 10:29:00 +00:00
|
|
|
if start {
|
|
|
|
CheckinProcess(s)
|
|
|
|
}
|
2020-06-06 00:36:39 +00:00
|
|
|
s.Failures = s.AllFailures().LastAmount(limitedFailures)
|
2020-03-04 10:29:00 +00:00
|
|
|
for _, c := range s.Checkins() {
|
|
|
|
s.AllCheckins = append(s.AllCheckins, c)
|
|
|
|
}
|
|
|
|
// collect initial service stats
|
|
|
|
s.UpdateStats()
|
2020-03-13 04:06:06 +00:00
|
|
|
allServices[s.Id] = s
|
2020-03-04 10:29:00 +00:00
|
|
|
}
|
|
|
|
return allServices, nil
|
|
|
|
}
|
|
|
|
|
2020-03-06 22:18:06 +00:00
|
|
|
func (s *Service) UpdateStats() *Service {
|
2020-03-04 10:29:00 +00:00
|
|
|
s.Online24Hours = s.OnlineDaysPercent(1)
|
|
|
|
s.Online7Days = s.OnlineDaysPercent(7)
|
|
|
|
s.AvgResponse = s.AvgTime()
|
2020-03-10 05:24:35 +00:00
|
|
|
s.FailuresLast24Hours = s.FailuresSince(utils.Now().Add(-time.Hour * 24)).Count()
|
2020-03-06 22:18:06 +00:00
|
|
|
|
2020-06-06 00:36:39 +00:00
|
|
|
allFails := s.AllFailures()
|
2020-03-06 22:18:06 +00:00
|
|
|
if s.LastOffline.IsZero() {
|
2020-06-06 00:36:39 +00:00
|
|
|
lastFail := allFails.Last()
|
2020-03-06 22:18:06 +00:00
|
|
|
if lastFail != nil {
|
2020-03-10 05:24:35 +00:00
|
|
|
s.LastOffline = lastFail.CreatedAt
|
2020-03-06 22:18:06 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-04 10:29:00 +00:00
|
|
|
s.Stats = &Stats{
|
2020-06-06 00:36:39 +00:00
|
|
|
Failures: allFails.Count(),
|
2020-03-04 10:29:00 +00:00
|
|
|
Hits: s.AllHits().Count(),
|
2020-03-06 22:18:06 +00:00
|
|
|
FirstHit: s.FirstHit().CreatedAt,
|
2020-03-04 10:29:00 +00:00
|
|
|
}
|
2020-03-06 22:18:06 +00:00
|
|
|
return s
|
2020-03-04 10:29:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// AvgTime will return the average amount of time for a service to response back successfully
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) AvgTime() int64 {
|
2020-03-06 22:18:06 +00:00
|
|
|
return s.AllHits().Avg()
|
2020-03-04 10:29:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// OnlineDaysPercent returns the service's uptime percent within last 24 hours
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) OnlineDaysPercent(days int) float32 {
|
2020-03-10 05:24:35 +00:00
|
|
|
ago := utils.Now().Add(-time.Duration(days) * types.Day)
|
2020-03-04 10:29:00 +00:00
|
|
|
return s.OnlineSince(ago)
|
|
|
|
}
|
|
|
|
|
|
|
|
// OnlineSince accepts a time since parameter to return the percent of a service's uptime.
|
|
|
|
func (s *Service) OnlineSince(ago time.Time) float32 {
|
2020-06-06 00:36:39 +00:00
|
|
|
failsList := s.FailuresSince(ago).Count()
|
|
|
|
hitsList := s.HitsSince(ago).Count()
|
2020-03-13 04:06:06 +00:00
|
|
|
|
|
|
|
if failsList == 0 {
|
2020-03-04 10:29:00 +00:00
|
|
|
s.Online24Hours = 100.00
|
|
|
|
return s.Online24Hours
|
|
|
|
}
|
2020-03-13 04:06:06 +00:00
|
|
|
|
|
|
|
if hitsList == 0 {
|
2020-03-04 10:29:00 +00:00
|
|
|
s.Online24Hours = 0
|
|
|
|
return s.Online24Hours
|
|
|
|
}
|
2020-03-13 04:06:06 +00:00
|
|
|
|
|
|
|
avg := (float64(failsList) / float64(hitsList)) * 100
|
2020-03-04 10:29:00 +00:00
|
|
|
avg = 100 - avg
|
|
|
|
if avg < 0 {
|
|
|
|
avg = 0
|
|
|
|
}
|
|
|
|
amount, _ := strconv.ParseFloat(fmt.Sprintf("%0.2f", avg), 10)
|
|
|
|
s.Online24Hours = float32(amount)
|
|
|
|
return s.Online24Hours
|
|
|
|
}
|
|
|
|
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) Uptime() utils.Duration {
|
2020-06-19 09:14:40 +00:00
|
|
|
return utils.Duration{Duration: utils.Now().Sub(s.LastOffline)}
|
|
|
|
}
|
2020-03-06 09:33:46 +00:00
|
|
|
|
2020-06-19 09:14:40 +00:00
|
|
|
// Downtime returns the amount of time of a offline service
|
2020-06-26 04:08:12 +00:00
|
|
|
func (s Service) Downtime() utils.Duration {
|
2020-06-19 09:14:40 +00:00
|
|
|
return utils.Duration{Duration: utils.Now().Sub(s.LastOnline)}
|
2020-03-04 10:29:00 +00:00
|
|
|
}
|
2020-06-06 00:36:39 +00:00
|
|
|
|
|
|
|
// ServiceOrder will reorder the services based on 'order_id' (Order)
|
|
|
|
type ServiceOrder []Service
|
|
|
|
|
|
|
|
// Sort interface for resroting the Services in order
|
|
|
|
func (c ServiceOrder) Len() int { return len(c) }
|
|
|
|
func (c ServiceOrder) Swap(i, j int) { c[int64(i)], c[int64(j)] = c[int64(j)], c[int64(i)] }
|
|
|
|
func (c ServiceOrder) Less(i, j int) bool { return c[i].Order < c[j].Order }
|