From c9dd23b36e49cbd586b415819134b3966a70fc3c Mon Sep 17 00:00:00 2001 From: Hunter Long Date: Sat, 23 Jun 2018 23:17:31 -0700 Subject: [PATCH] updates - prometheus exporter /metrics --- README.md | 22 +++++++++++++++++----- checker.go | 2 +- database.go | 18 ++++++++++++++++++ failures.go | 7 +++++-- main.go | 11 +++++++++++ web.go | 31 +++++++++++++++++++++++++++++-- 6 files changed, 81 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 28f26a8b..5cc3d99d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ If your server crashes your Status Page should still remaining online to notify ## Run on Docker Use the [Statup Docker Image](https://hub.docker.com/r/hunterlong/statup) to create a status page in seconds. -``` +```bash docker run -it -p 8080:8080 hunterlong/statup ``` There are multiple way to startup a Statup server. You want to make sure Statup is on it's own instance that is not on the same server as the applications you wish to monitor. @@ -16,13 +16,13 @@ It doesn't look good when your Status Page goes down, I recommend a small EC2 in ## Docker Compose In this folder there is a standard docker-compose file that include nginx, postgres, and Statup. -```$xslt +```bash docker-compose up -d ``` ## Docker Compose with Automatic SSL You can automatically start a Statup server with automatic SSL encryption using this docker-compose file. First point your domain's DNS to the Statup server, and then run this docker-compose command with DOMAIN and EMAIL. Email is for letsencrypt services. -``` +```bash LETSENCRYPT_HOST=mydomain.com \ LETSENCRYPT_EMAIL=info@mydomain.com \ docker-compose -f docker-compose-ssl.yml up -d @@ -31,7 +31,7 @@ Once your instance has started, it will take a moment to get your SSL certificat ## Run on AWS EC2 Running Statup on the smallest EC2 server is very quick using the AWS AMI Image: `ami-1f7c3567`. -``` +```bash aws ec2 run-instances \ --image-id ami-1f7c3567 \ --count 1 \ @@ -49,4 +49,16 @@ Statup includes email notification via SMTP if your services go offline. ## User Created Plugins Statup isn't just another Status Page for your applications, it's a framework that allows you to create your own plugins to interact with every element of your status page. Plugin are created in Golang using the [statup/plugin](https://github.com/hunterlong/statup/tree/master/plugin) golang package. The plugin package has a list of -interfaces/events to accept into your own plugin application. \ No newline at end of file +interfaces/events to accept into your own plugin application. + +## Prometheus Exporter +Statup includes a prometheus exporter so you can have even more monitoring power with your services. The prometheus exporter can be seen on `/metrics`, simply create another exporter in your prometheus config. +```yaml +scrape_configs: + - job_name: 'statup' + static_configs: + - targets: ['statup:8080'] +``` + + + diff --git a/checker.go b/checker.go index 92feeecf..645ddb13 100644 --- a/checker.go +++ b/checker.go @@ -23,7 +23,7 @@ func (s *Service) CheckQueue() { if s.Interval < 1 { s.Interval = 1 } - fmt.Printf(" Service: %v | Online: %v | Latency: %v\n", s.Name, s.Online, s.Latency) + fmt.Printf(" Service: %v | Online: %v | Latency: %0.0fms\n", s.Name, s.Online, (s.Latency * 100)) time.Sleep(time.Duration(s.Interval) * time.Second) } diff --git a/database.go b/database.go index 5d3ee7d0..e5594727 100644 --- a/database.go +++ b/database.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "time" + "upper.io/db.v3" "upper.io/db.v3/lib/sqlbuilder" "upper.io/db.v3/mysql" "upper.io/db.v3/postgresql" @@ -62,6 +64,22 @@ func DbConnection(dbType string) error { return err } +func DatabaseMaintence() { + defer DatabaseMaintence() + since := time.Now().AddDate(0, 0, -7) + DeleteAllSince("failures", since) + DeleteAllSince("hits", since) + time.Sleep(60 * time.Minute) +} + +func DeleteAllSince(table string, date time.Time) { + sql := fmt.Sprintf("DELETE FROM %v WHERE created_at < '%v';", table, date.Format("2006-01-02")) + _, err := dbSession.Exec(db.Raw(sql)) + if err != nil { + fmt.Println(err) + } +} + func Backup() { } diff --git a/failures.go b/failures.go index 9a7f554f..b1ad457d 100644 --- a/failures.go +++ b/failures.go @@ -61,10 +61,13 @@ func (f *Failure) Delete() error { return col.Delete() } -func CountFailures() (uint64, error) { +func CountFailures() uint64 { col := dbSession.Collection("failures").Find() amount, err := col.Count() - return amount, err + if err != nil { + return 0 + } + return amount } func (s *Service) TotalFailures() (uint64, error) { diff --git a/main.go b/main.go index b11c8a94..448b56d7 100644 --- a/main.go +++ b/main.go @@ -123,6 +123,13 @@ func DownloadFile(filepath string, url string) error { //} func main() { + if len(os.Args) >= 2 { + if os.Args[1] == "version" { + fmt.Printf("Statup v%v\n", VERSION) + } + os.Exit(0) + } + var err error fmt.Printf("Starting Statup v%v\n", VERSION) @@ -153,9 +160,13 @@ func mainProcess() { fmt.Println("Core database was not found, Statup is not setup yet.") RunHTTPServer() } + CheckServices() core.Communications, _ = SelectAllCommunications() LoadDefaultCommunications() + + go DatabaseMaintence() + if !setupMode { LoadPlugins() RunHTTPServer() diff --git a/web.go b/web.go index 75a9806d..84f3c1b4 100644 --- a/web.go +++ b/web.go @@ -58,6 +58,7 @@ func Router() *mux.Router { r.Handle("/api/services/{id}", http.HandlerFunc(ApiServiceUpdateHandler)).Methods("POST") r.Handle("/api/users", http.HandlerFunc(ApiAllUsersHandler)) r.Handle("/api/users/{id}", http.HandlerFunc(ApiUserHandler)) + r.Handle("/metrics", http.HandlerFunc(PrometheusHandler)).Methods("GET") return r } @@ -146,7 +147,6 @@ func CreateServiceHandler(w http.ResponseWriter, r *http.Request) { Type: checkType, Port: port, } - _, err := service.Create() if err != nil { go service.CheckQueue() @@ -154,6 +154,30 @@ func CreateServiceHandler(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/services", http.StatusSeeOther) } +func PrometheusHandler(w http.ResponseWriter, r *http.Request) { + fmt.Printf("Prometheus /metrics Request From IP: %v\n", r.RemoteAddr) + metrics := []string{} + system := fmt.Sprintf("statup_total_failures %v\n", CountFailures()) + system += fmt.Sprintf("statup_total_services %v", len(services)) + metrics = append(metrics, system) + + for _, v := range services { + online := 1 + if !v.Online { + online = 0 + } + met := fmt.Sprintf("statup_service_failures{id=\"%v\" name=\"%v\"} %v\n", v.Id, v.Name, len(v.Failures)) + met += fmt.Sprintf("statup_service_latency{id=\"%v\" name=\"%v\"} %0.0f\n", v.Id, v.Name, (v.Latency * 100)) + met += fmt.Sprintf("statup_service_online{id=\"%v\" name=\"%v\"} %v\n", v.Id, v.Name, online) + met += fmt.Sprintf("statup_service_status_code{id=\"%v\" name=\"%v\"} %v\n", v.Id, v.Name, v.LastStatusCode) + met += fmt.Sprintf("statup_service_response_length{id=\"%v\" name=\"%v\"} %v", v.Id, v.Name, len([]byte(v.LastResponse))) + metrics = append(metrics, met) + } + output := strings.Join(metrics, "\n") + w.WriteHeader(http.StatusOK) + w.Write([]byte(output)) +} + func SetupHandler(w http.ResponseWriter, r *http.Request) { if core != nil { http.Redirect(w, r, "/", http.StatusSeeOther) @@ -210,7 +234,7 @@ func DashboardHandler(w http.ResponseWriter, r *http.Request) { err := ErrorResponse{} ExecuteResponse(w, r, "login.html", err) } else { - fails, _ := CountFailures() + fails := CountFailures() out := dashboard{services, core, CountOnline(), len(services), fails} ExecuteResponse(w, r, "dashboard.html", out) } @@ -257,6 +281,9 @@ func ServicesDeleteFailuresHandler(w http.ResponseWriter, r *http.Request) { } func IsAuthenticated(r *http.Request) bool { + if core == nil { + return false + } if store == nil { return false }