service date range - database query updates - larger seed data

Makefile

@@ -1,4 +1,4 @@
-VERSION=0.65
+VERSION=0.66
 BINARY_NAME=statup
 GOPATH:=$(GOPATH)
 GOCMD=go

cmd/main.go

@@ -59,6 +59,7 @@ func main() {
 	source.Assets()
 	utils.InitLogs()
 	args := flag.Args()
+	defer core.CloseDB()
 
 	if len(args) >= 1 {
 		err := CatchCLI(args)

core/database.go

@@ -78,6 +78,17 @@ type DbConfig struct {
 	*types.DbConfig
 }
 
+func (s *Service) HitsBetween(t1, t2 time.Time) *gorm.DB {
+	selector := Dbtimestamp(3600)
+	return DbSession.Debug().Model(&types.Hit{}).Select(selector).Where("service = ? AND created_at BETWEEN ? AND ?", s.Id, t1.UTC().Format(types.TIME), t2.UTC().Format(types.TIME)).Group("timeframe")
+}
+
+func CloseDB() {
+	if DbSession != nil {
+		DbSession.DB().Close()
+	}
+}
+
 // Close shutsdown the database connection
 func (db *DbConfig) Close() error {
 	return DbSession.DB().Close()
@@ -104,7 +115,9 @@ func (u *User) AfterFind() (err error) {
 }
 
 func (u *Hit) BeforeCreate() (err error) {
+	if u.CreatedAt.IsZero() {
 		u.CreatedAt = time.Now().UTC()
+	}
 	return
 }
 

core/sample.go

@@ -97,10 +97,10 @@ func InsertSampleHits() error {
 		utils.Log(1, fmt.Sprintf("Adding %v sample hit records to service %v", 360, service.Name))
 		createdAt := since
 
-		for hi := int64(1); hi <= 860; hi++ {
+		for hi := int64(1); hi <= 1860; hi++ {
 			rand.Seed(time.Now().UnixNano())
 			latency := rand.Float64()
-			createdAt = createdAt.Add(15 * time.Minute)
+			createdAt = createdAt.Add(3 * time.Minute).UTC()
 			hit := &types.Hit{
 				Service:   service.Id,
 				CreatedAt: createdAt,

core/services.go

@@ -122,7 +122,7 @@ func (s *Service) OnlineSince(ago time.Time) float32 {
 
 // DateScan struct is for creating the charts.js graph JSON array
 type DateScan struct {
-	CreatedAt time.Time `json:"x"`
+	CreatedAt string `json:"x"`
 	Value     int64  `json:"y"`
 }
 
@@ -167,19 +167,45 @@ func (s *Service) DowntimeText() string {
 }
 
 // GroupDataBy returns a SQL query as a string to group a column by a time
-func GroupDataBy(column string, id int64, start, end time.Time, increment string) string {
+func GroupDataBy(column string, id int64, start, end time.Time, seconds int64) string {
+	incrementTime := "second"
+	if seconds == 60 {
+		incrementTime = "minute"
+	} else if seconds == 3600 {
+		incrementTime = "hour"
+	}
 	var sql string
 	switch CoreApp.DbConnection {
 	case "mysql":
 		sql = fmt.Sprintf("SELECT CONCAT(date_format(created_at, '%%Y-%%m-%%dT%%H:%%i:00Z')) AS created_at, AVG(latency)*1000 AS value FROM %v WHERE service=%v AND DATE_FORMAT(created_at, '%%Y-%%m-%%dT%%TZ') BETWEEN DATE_FORMAT('%v', '%%Y-%%m-%%dT%%TZ') AND DATE_FORMAT('%v', '%%Y-%%m-%%dT%%TZ') GROUP BY 1 ORDER BY created_at ASC;", column, id, start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339))
 	case "sqlite":
-		sql = fmt.Sprintf("SELECT strftime('%%Y-%%m-%%dT%%H:%%M:00Z', created_at), AVG(latency)*1000 as value FROM %v WHERE service=%v AND created_at >= '%v' AND created_at <= '%v' GROUP BY strftime('%%M:00', created_at) ORDER BY created_at ASC;", column, id, start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339))
+		sql = fmt.Sprintf("SELECT datetime((strftime('%%s', created_at) / %v) * %v, 'unixepoch'), AVG(latency)*1000 as value FROM %v WHERE service=%v AND created_at BETWEEN '%v' AND '%v' GROUP BY 1 ORDER BY created_at ASC;", seconds, seconds, column, id, start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339))
 	case "postgres":
-		sql = fmt.Sprintf("SELECT date_trunc('%v', created_at), AVG(latency)*1000 AS value FROM %v WHERE service=%v AND created_at >= '%v' AND created_at <= '%v' GROUP BY 1 ORDER BY date_trunc ASC;", increment, column, id, start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339))
+		sql = fmt.Sprintf("SELECT date_trunc('%v', created_at), AVG(latency)*1000 AS value FROM %v WHERE service=%v AND created_at >= '%v' AND created_at <= '%v' GROUP BY 1 ORDER BY date_trunc ASC;", incrementTime, column, id, start.UTC().Format(time.RFC3339), end.UTC().Format(time.RFC3339))
 	}
+	fmt.Println(sql)
 	return sql
 }
 
+func Dbtimestamp(seconds int64) string {
+	incrementTime := "second"
+	if seconds == 60 {
+		incrementTime = "minute"
+	} else if seconds == 3600 {
+		incrementTime = "hour"
+	}
+	switch CoreApp.DbConnection {
+	case "mysql":
+		return fmt.Sprintf("CONCAT(date_format(created_at, '%%Y-%%m-%%d %%H:00:00')) AS timeframe, AVG(latency) AS value")
+	case "sqlite":
+		return fmt.Sprintf("datetime((strftime('%%s', created_at) / %v) * %v, 'unixepoch') AS timeframe, AVG(latency) as value", seconds, seconds)
+	case "postgres":
+		return fmt.Sprintf("date_trunc('%v', created_at) AS timeframe, AVG(latency) AS value", incrementTime)
+	default:
+		return ""
+	}
+}
+
 // Downtime returns the amount of time of a offline service
 func (s *Service) Downtime() time.Duration {
 	hits, _ := s.Hits()
@@ -196,27 +222,21 @@ func (s *Service) Downtime() time.Duration {
 
 func GraphDataRaw(service types.ServiceInterface, start, end time.Time) *DateScanObj {
 	var d []DateScan
-	s := service.Select()
+	//s := service.Select()
-	sql := GroupDataBy("hits", s.Id, start, end, "minute")
+
-	rows, err := DbSession.Raw(sql).Rows()
+	model := service.(*Service).HitsBetween(start, end)
-	if err != nil {
+	rows, _ := model.Rows()
-		utils.Log(2, err)
+
-		return nil
+	//sql := GroupDataBy("hits", s.Id, start, end, 3600)
-	}
 	for rows.Next() {
 		var gd DateScan
-		var tt string
+		var createdAt string
-		var ff float64
+		var value float64
-		err := rows.Scan(&tt, &ff)
+		rows.Scan(&createdAt, &value)
-		if err != nil {
+
-			utils.Log(2, fmt.Sprintf("Issue loading chart data for service %v, %v", s.Name, err))
+		createdTime, _ := time.Parse(types.TIME, createdAt)
-		}
+		gd.CreatedAt = utils.Timezoner(createdTime, CoreApp.Timezone).Format(types.TIME)
-		gd.CreatedAt, err = time.Parse(time.RFC3339, tt)
+		gd.Value = int64(value * 1000)
-		if err != nil {
-			utils.Log(2, fmt.Sprintf("Issue parsing time %v", err))
-		}
-		gd.CreatedAt = utils.Timezoner(gd.CreatedAt, CoreApp.Timezone)
-		gd.Value = int64(ff)
 		d = append(d, gd)
 	}
 	return &DateScanObj{d}
@@ -233,7 +253,7 @@ func (d *DateScanObj) ToString() string {
 
 // GraphData returns the JSON object used by Charts.js to render the chart
 func (s *Service) GraphData() string {
-	start := time.Now().Add(time.Hour*-24 + time.Minute*0 + time.Second*0)
+	start := time.Now().Add(-24 * time.Hour)
 	end := time.Now()
 	obj := GraphDataRaw(s, start, end)
 	data, err := json.Marshal(obj)

core/services_test.go

@@ -340,25 +340,3 @@ func TestDNScheckService(t *testing.T) {
 	assert.Nil(t, err)
 	assert.NotZero(t, amount)
 }
-
-func TestGroupGraphData(t *testing.T) {
-	service := SelectService(1)
-	CoreApp.DbConnection = "mysql"
-	lastWeek := time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
-	out := GroupDataBy("services", service.Id, lastWeek, time.Now(), "hour")
-	t.Log(out)
-	assert.Contains(t, out, "SELECT CONCAT(date_format(created_at, '%Y-%m-%dT%H:%i:00Z'))")
-
-	CoreApp.DbConnection = "postgres"
-	lastWeek = time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
-	out = GroupDataBy("services", service.Id, lastWeek, time.Now(), "hour")
-	t.Log(out)
-	assert.Contains(t, out, "SELECT date_trunc('hour', created_at)")
-
-	CoreApp.DbConnection = "sqlite"
-	lastWeek = time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
-	out = GroupDataBy("services", service.Id, lastWeek, time.Now(), "hour")
-	t.Log(out)
-	assert.Contains(t, out, "SELECT strftime('%Y-%m-%dT%H:%M:00Z'")
-
-}

handlers/dashboard.go

@@ -16,9 +16,11 @@
 package handlers
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/hunterlong/statup/core"
 	"github.com/hunterlong/statup/source"
+	"github.com/hunterlong/statup/types"
 	"github.com/hunterlong/statup/utils"
 	"net/http"
 )
@@ -94,3 +96,19 @@ func logsLineHandler(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(lastLine.FormatForHtml()))
 	}
 }
+
+type exportData struct {
+	Services []types.ServiceInterface
+}
+
+func exportHandler(w http.ResponseWriter, r *http.Request) {
+	if !IsAuthenticated(r) {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	data := exportData{core.CoreApp.Services}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(data)
+}

handlers/routes.go

@@ -74,6 +74,7 @@ func Router() *mux.Router {
 	r.Handle("/settings/build", http.HandlerFunc(saveAssetsHandler)).Methods("GET")
 	r.Handle("/settings/delete_assets", http.HandlerFunc(deleteAssetsHandler)).Methods("GET")
 	r.Handle("/settings/notifier/{method}", http.HandlerFunc(saveNotificationHandler)).Methods("POST")
+	r.Handle("/settings/export", http.HandlerFunc(exportHandler)).Methods("GET")
 	r.Handle("/plugins/download/{name}", http.HandlerFunc(pluginsDownloadHandler))
 	r.Handle("/plugins/{name}/save", http.HandlerFunc(pluginSavedHandler)).Methods("POST")
 	r.Handle("/help", http.HandlerFunc(helpHandler))

handlers/services.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hunterlong/statup/core"
 	"github.com/hunterlong/statup/types"
 	"github.com/hunterlong/statup/utils"
+	"github.com/jinzhu/now"
 	"net/http"
 	"strconv"
 	"time"
@@ -39,19 +40,21 @@ func renderServiceChartHandler(w http.ResponseWriter, r *http.Request) {
 
 	startField := fields.Get("start")
 	endField := fields.Get("end")
-	var start time.Time
+
-	var end time.Time
+	end := now.EndOfDay().UTC()
-	if startField == "" {
+	start := now.BeginningOfDay().UTC()
-		start = time.Now().Add((-24 * 7) * time.Hour).UTC()
+
-	} else {
+	if startField != "" {
-		start = time.Unix(utils.StringInt(startField), 0).UTC()
+		start = time.Unix(utils.StringInt(startField), 0)
+		start = now.New(start).BeginningOfDay().UTC()
 	}
-	if endField == "" {
+	if endField != "" {
-		end = time.Now().UTC()
+		end = time.Unix(utils.StringInt(endField), 0)
-	} else {
+		end = now.New(end).EndOfDay().UTC()
-		end = time.Unix(utils.StringInt(endField), 0).UTC()
 	}
 
+	fmt.Println("start: ", start.String(), "end: ", end.String())
+
 	service := core.SelectService(utils.StringInt(vars["id"]))
 	data := core.GraphDataRaw(service, start, end).ToString()
 
@@ -69,8 +72,9 @@ func renderServiceChartsHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "max-age=60")
 
 	var data []string
-	end := time.Now()
+	end := now.EndOfDay().UTC()
-	start := end.Add(-(24 * 7) * time.Hour)
+	start := now.BeginningOfDay().UTC()
+
 	for _, s := range services {
 		d := core.GraphDataRaw(s, start, end).ToString()
 		data = append(data, d)
@@ -106,7 +110,6 @@ func reorderServiceHandler(w http.ResponseWriter, r *http.Request) {
 	decoder := json.NewDecoder(r.Body)
 	decoder.Decode(&newOrder)
 	for _, s := range newOrder {
-		fmt.Println("updating: ", s.Id, " to be order_id: ", s.Order)
 		service := core.SelectService(s.Id)
 		service.Order = s.Order
 		service.Update(false)
@@ -183,16 +186,24 @@ func servicesViewHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if startField == 0 || endField == 0 {
+	end := time.Now()
-		startField = time.Now().Add((-24 * 7) * time.Hour).UTC().Unix()
+	start := end.Add((-24 * 7) * time.Hour)
-		endField = time.Now().UTC().Unix()
+
+	if startField != 0 {
+		start = time.Unix(startField, 0)
 	}
+	if endField != 0 {
+		end = time.Unix(endField, 0)
+	}
+
+	data := core.GraphDataRaw(serv, start, end)
 
 	out := struct {
 		Service *core.Service
 		Start   int64
 		End     int64
-	}{serv, startField, endField}
+		Data    string
+	}{serv, start.Unix(), end.Unix(), data.ToString()}
 
 	executeResponse(w, r, "service.html", out, nil)
 }

source/tmpl/service.html

@@ -57,7 +57,7 @@
                             </div>
                         </div>
 
-                <div class="chart-container" style="height: 250px">
+                <div class="chart-container" style="height: 400px">
                     <canvas id="service"></canvas>
                 </div>
 
@@ -255,7 +255,7 @@
         data: {
             datasets: [{
                 label: 'Response Time (Milliseconds)',
-                data: {{js .GraphData}},
+                data: {{js .Data}},
                 backgroundColor: [
                     'rgba(47, 206, 30, 0.92)'
                 ],
@@ -275,14 +275,13 @@
                         beginAtZero: true
                     },
                     gridLines: {
-                        display:false
+                        display: true
                     }
                 }],
                 xAxes: [{
                     type: 'time',
-                    distribution: 'series',
                     gridLines: {
-                        display:false
+                        display:true
                     }
                 }]
             },

types/time.go

@@ -19,6 +19,12 @@ import (
 	"time"
 )
 
+const (
+	TIME_NANOZ = "2006-01-02 15:04:05.999999-0700 MST"
+	TIME_NANO  = "2006-01-02T15:04:05Z"
+	TIME       = "2006-01-02 15:04:05"
+)
+
 var (
 	NOW         = func() time.Time { return time.Now() }()
 	HOUR_1_AGO  = time.Now().Add(-1 * time.Hour)