From 7dc285295b28b938f338aa419cf8dc2ae31f8c8c Mon Sep 17 00:00:00 2001 From: Hunter Long Date: Fri, 6 Mar 2020 01:33:46 -0800 Subject: [PATCH] db changes --- Makefile | 6 ++ database/database.go | 17 ++++- database/grouping.go | 63 +++++++++---------- database/time.go | 7 ++- dev/docker-compose.db.yml | 2 +- dev/prometheus.yml | 6 ++ frontend/src/API.js | 8 +-- frontend/src/assets/scss/base.scss | 22 +++++++ frontend/src/assets/scss/mobile.scss | 6 +- .../Dashboard/DashboardMessages.vue | 2 +- .../Dashboard/DashboardServices.vue | 4 +- .../components/Dashboard/DashboardUsers.vue | 6 +- .../components/Index/GroupServiceFailures.vue | 2 +- .../src/components/Service/ServiceChart.vue | 6 +- .../src/components/Service/ServiceHeatmap.vue | 2 +- .../src/components/Service/ServiceInfo.vue | 4 +- frontend/src/forms/Message.vue | 4 +- frontend/src/forms/User.vue | 4 +- frontend/src/pages/Index.vue | 2 +- frontend/src/pages/Service.vue | 2 +- handlers/handlers.go | 2 +- handlers/services.go | 4 +- handlers/setup.go | 1 + types/configs/connection.go | 2 +- types/configs/migration.go | 4 ++ types/failures/samples.go | 4 ++ types/hits/interface.go | 16 ++++- types/hits/samples.go | 59 ++++++++++------- types/incidents/database_updates.go | 6 +- types/interface.go | 1 - types/messages/database.go | 6 +- types/services/failures.go | 15 +++-- types/services/hits.go | 8 +++ types/services/methods.go | 14 ++--- types/users/auth.go | 5 -- types/users/database.go | 25 +++++--- 36 files changed, 221 insertions(+), 126 deletions(-) delete mode 100644 types/interface.go diff --git a/Makefile b/Makefile index b6a73fc2..3211c429 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,12 @@ stop: logs: docker logs statping --follow +db-up: + docker-compose -f dev/docker-compose.db.yml up -d --remove-orphans + +db-down: + docker-compose -f dev/docker-compose.full.yml down --remove-orphans + console: docker exec -t -i statping /bin/sh diff --git a/database/database.go b/database/database.go index faaabfc1..ee02446a 100644 --- a/database/database.go +++ b/database/database.go @@ -104,17 +104,22 @@ type Database interface { Since(time.Time) Database Between(time.Time, time.Time) Database - SelectByTime(string) string + SelectByTime(time.Duration) string MultipleSelects(args ...string) Database FormatTime(t time.Time) string ParseTime(t string) (time.Time, error) + DbType() string } func DB() Database { return database } +func (it *Db) DbType() string { + return it.Database.Dialect().GetName() +} + func Close() error { if database == nil { return nil @@ -145,11 +150,21 @@ func Available() bool { return true } +func AmountGreaterThan1000(db *gorm.DB) *gorm.DB { + return db.Where("service = ?", 1000) +} + func (it *Db) MultipleSelects(args ...string) Database { joined := strings.Join(args, ", ") return it.Select(joined) } +func OrderStatus(status []string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + return db.Scopes(AmountGreaterThan1000).Where("status IN (?)", status) + } +} + type Db struct { Database *gorm.DB Type string diff --git a/database/grouping.go b/database/grouping.go index 2a16332c..d3bec6ff 100644 --- a/database/grouping.go +++ b/database/grouping.go @@ -28,7 +28,7 @@ func (b By) String() string { type GroupQuery struct { Start time.Time End time.Time - Group string + Group time.Duration Order string Limit int Offset int @@ -47,8 +47,15 @@ func (b GroupQuery) Database() Database { var ( ByCount = By("COUNT(id) as amount") - ByAverage = func(column string) By { - return By(fmt.Sprintf("AVG(%s) as amount", column)) + ByAverage = func(column string, multiplier int) By { + switch database.DbType() { + case "mysql": + return By(fmt.Sprintf("CAST(AVG(%s)*%d as UNSIGNED) as amount", column, multiplier)) + case "postgres": + return By(fmt.Sprintf("cast(AVG(%s)*%d as int) as amount", column, multiplier)) + default: + return By(fmt.Sprintf("cast(AVG(%s)*%d as int) as amount", column, multiplier)) + } } ) @@ -76,7 +83,7 @@ func (g *GroupQuery) toFloatRows() []*TimeValue { fmt.Println("float rows: ", timeframe, amount) - newTs := types.FixedTime(timeframe, g.duration()) + newTs := types.FixedTime(timeframe, g.Group) data = append(data, &TimeValue{ Timeframe: newTs, Amount: amount, @@ -91,7 +98,7 @@ func (g *GroupQuery) GraphData(by By) ([]*TimeValue, error) { dbQuery := g.db.MultipleSelects( g.db.SelectByTime(g.Group), by.String(), - ).Group("timeframe") + ).Group("timeframe").Debug() g.db = dbQuery @@ -119,7 +126,7 @@ func (g *GroupQuery) ToTimeValue() (*TimeVar, error) { log.Error(err, timeframe) } trueTime, _ := g.db.ParseTime(timeframe) - newTs := types.FixedTime(trueTime, g.duration()) + newTs := types.FixedTime(trueTime, g.Group) data = append(data, &TimeValue{ Timeframe: newTs, Amount: amount, @@ -131,12 +138,12 @@ func (g *GroupQuery) ToTimeValue() (*TimeVar, error) { func (t *TimeVar) FillMissing(current, end time.Time) ([]*TimeValue, error) { timeMap := make(map[string]float64) var validSet []*TimeValue - dur := t.g.duration() + dur := t.g.Group for _, v := range t.data { timeMap[v.Timeframe] = v.Amount } - currentStr := types.FixedTime(current, t.g.duration()) + currentStr := types.FixedTime(current, t.g.Group) for { var amount float64 @@ -151,31 +158,12 @@ func (t *TimeVar) FillMissing(current, end time.Time) ([]*TimeValue, error) { break } current = current.Add(dur) - currentStr = types.FixedTime(current, t.g.duration()) + currentStr = types.FixedTime(current, t.g.Group) } return validSet, nil } -func (g *GroupQuery) duration() time.Duration { - switch g.Group { - case "second": - return types.Second - case "minute": - return types.Minute - case "hour": - return types.Hour - case "day": - return types.Day - case "month": - return types.Month - case "year": - return types.Year - default: - return types.Hour - } -} - type isObject interface { Db() Database } @@ -183,9 +171,6 @@ type isObject interface { func ParseQueries(r *http.Request, o isObject) *GroupQuery { fields := parseGet(r) grouping := fields.Get("group") - if grouping == "" { - grouping = "hour" - } startField := utils.ToInt(fields.Get("start")) endField := utils.ToInt(fields.Get("end")) limit := utils.ToInt(fields.Get("limit")) @@ -198,10 +183,19 @@ func ParseQueries(r *http.Request, o isObject) *GroupQuery { db := o.Db() + if grouping == "" { + grouping = "1h" + } + groupDur, err := time.ParseDuration(grouping) + if err != nil { + log.Errorln(err) + groupDur = 1 * time.Hour + } + query := &GroupQuery{ Start: time.Unix(startField, 0).UTC(), End: time.Unix(endField, 0).UTC(), - Group: grouping, + Group: groupDur, Order: orderBy, Limit: int(limit), Offset: int(offset), @@ -210,11 +204,14 @@ func ParseQueries(r *http.Request, o isObject) *GroupQuery { } if startField == 0 { - query.Start = time.Now().Add(-3 * types.Month).UTC() + query.Start = time.Now().Add(-7 * types.Day).UTC() } if endField == 0 { query.End = time.Now().UTC() } + if query.End.After(utils.Now()) { + query.End = utils.Now() + } if query.Limit != 0 { db = db.Limit(query.Limit) diff --git a/database/time.go b/database/time.go index 6c7bfacc..e8832e09 100644 --- a/database/time.go +++ b/database/time.go @@ -30,14 +30,15 @@ func (it *Db) FormatTime(t time.Time) string { } } -func (it *Db) SelectByTime(increment string) string { +func (it *Db) SelectByTime(increment time.Duration) string { + seconds := int(increment.Seconds()) switch it.Type { case "mysql": - return fmt.Sprintf("CONCAT(date_format(created_at, '%s')) AS timeframe", it.correctTimestamp(increment)) + return fmt.Sprintf("FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(created_at) / %d) * %d) AS timeframe", seconds, seconds) case "postgres": return fmt.Sprintf("date_trunc('%s', created_at) AS timeframe", increment) default: - return fmt.Sprintf("strftime('%s', created_at, 'utc') as timeframe", it.correctTimestamp(increment)) + return fmt.Sprintf("datetime((strftime('%%s', created_at) / %d) * %d, 'unixepoch') as timeframe", seconds, seconds) } } diff --git a/dev/docker-compose.db.yml b/dev/docker-compose.db.yml index 02011a4b..6588a813 100644 --- a/dev/docker-compose.db.yml +++ b/dev/docker-compose.db.yml @@ -29,7 +29,7 @@ services: MYSQL_ROOT_PASSWORD: password123 MYSQL_DATABASE: statping MYSQL_USER: root - MYSQL_PASSWORD: password + MYSQL_PASSWORD: password123 ports: - 3306:3306 healthcheck: diff --git a/dev/prometheus.yml b/dev/prometheus.yml index 7421f7f9..8637d797 100644 --- a/dev/prometheus.yml +++ b/dev/prometheus.yml @@ -3,6 +3,12 @@ global: evaluation_interval: 15s scrape_configs: + - job_name: 'statping_local' + scrape_interval: 15s + bearer_token: 'samplesecret' + static_configs: + - targets: ['docker0:8585'] + - job_name: 'statping' scrape_interval: 15s bearer_token: 'exampleapisecret' diff --git a/frontend/src/API.js b/frontend/src/API.js index c99e0a68..0b857f05 100644 --- a/frontend/src/API.js +++ b/frontend/src/API.js @@ -36,12 +36,12 @@ class Api { return axios.post('/api/services/' + data.id, data).then(response => (response.data)) } - async service_hits(id, start, end, group) { - return axios.get('/api/services/' + id + '/hits_data?start=' + start + '&end=' + end + '&group=' + group + '&fill=true').then(response => (response.data)) + async service_hits(id, start, end, group, fill=true) { + return axios.get('/api/services/' + id + '/hits_data?start=' + start + '&end=' + end + '&group=' + group + '&fill=' + fill).then(response => (response.data)) } - async service_failures_data(id, start, end, group) { - return axios.get('/api/services/' + id + '/failure_data?start=' + start + '&end=' + end + '&group=' + group + '&fill=true').then(response => (response.data)) + async service_failures_data(id, start, end, group, fill=true) { + return axios.get('/api/services/' + id + '/failure_data?start=' + start + '&end=' + end + '&group=' + group + '&fill=' + fill).then(response => (response.data)) } async service_heatmap(id, start, end, group) { diff --git a/frontend/src/assets/scss/base.scss b/frontend/src/assets/scss/base.scss index 0cd79339..b2dfbd09 100644 --- a/frontend/src/assets/scss/base.scss +++ b/frontend/src/assets/scss/base.scss @@ -539,4 +539,26 @@ input.inputTags-field:focus { cursor: pointer; } +.list-group-item { + min-height: 85pt; +} + +.index_container { + min-height: 980pt; +} + +/* Enter and leave animations can use different */ +/* durations and timing functions. */ +.slide-fade-enter-active { + transition: all .3s ease; +} +.slide-fade-leave-active { + transition: all .8s cubic-bezier(1.0, 0.5, 0.8, 1.0); +} +.slide-fade-enter, .slide-fade-leave-to +/* .slide-fade-leave-active below version 2.1.8 */ { + transform: translateX(10px); + opacity: 0; +} + @import 'mobile'; diff --git a/frontend/src/assets/scss/mobile.scss b/frontend/src/assets/scss/mobile.scss index 98fbf2b1..5db570e9 100644 --- a/frontend/src/assets/scss/mobile.scss +++ b/frontend/src/assets/scss/mobile.scss @@ -17,7 +17,8 @@ .container { padding: 0px !important; - padding-top: 15px !important; + padding-top: 4vh !important; + min-height: 960pt; } .group_header { @@ -29,6 +30,9 @@ margin-top: 0px; width: 100%; margin-bottom: 0; + box-shadow: 1px 10px 20px 0px; + height: 55pt; + color: rgba(0, 0, 0, 0.20); } .btn-sm { diff --git a/frontend/src/components/Dashboard/DashboardMessages.vue b/frontend/src/components/Dashboard/DashboardMessages.vue index bd5841fe..b72dc84e 100644 --- a/frontend/src/components/Dashboard/DashboardMessages.vue +++ b/frontend/src/components/Dashboard/DashboardMessages.vue @@ -1,7 +1,7 @@