reduce cpu usage on services - removed JS from assets - added debug build for benchmarking - tests

Makefile

@@ -1,4 +1,4 @@
-VERSION=0.51
+VERSION=0.52
 BINARY_NAME=statup
 GOPATH:=$(GOPATH)
 GOCMD=go
@@ -29,6 +29,9 @@ docker-publish-all: docker-push-base docker-push-dev docker-push-latest
 build: compile
 	$(GOBUILD) $(BUILDVERSION) -o $(BINARY_NAME) -v ./cmd
 
+build-debug: compile
+	$(GOBUILD) $(BUILDVERSION) -tags debug -o $(BINARY_NAME) -v ./cmd
+
 install: build
 	mv $(BINARY_NAME) $(GOPATH)/bin/$(BINARY_NAME)
 	$(GOPATH)/bin/$(BINARY_NAME) version
@@ -41,6 +44,12 @@ compile:
 	sass source/scss/base.scss source/css/base.css
 	rm -rf .sass-cache
 
+benchmark:
+	cd handlers && go test -v -run=^$ -bench=. -benchtime=5s -memprofile=prof.mem -cpuprofile=prof.cpu
+
+benchmark-view:
+	go tool pprof handlers/handlers.test handlers/prof.cpu > top20
+
 test: clean compile install
 	STATUP_DIR=$(TEST_DIR) go test -v -p=1 $(BUILDVERSION) -coverprofile=coverage.out ./...
 	gocov convert coverage.out > coverage.json
@@ -145,6 +154,10 @@ clean:
 	rm -rf utils/{logs,assets,plugins,statup.db,config.yml,.sass-cache,*.log}
 	rm -rf dev/test/cypress/videos
 	rm -f coverage.* sass
+	find . -name "*.out" -type f -delete
+	find . -name "*.cpu" -type f -delete
+	find . -name "*.mem" -type f -delete
+	find . -name "*.test" -type f -delete
 
 tag:
 	git tag "v$(VERSION)" --force

cmd/cli.go

@@ -39,12 +39,12 @@ const (
 )
 
 func CatchCLI(args []string) error {
-	utils.InitLogs()
 	dir := utils.Directory
+	utils.InitLogs()
 	source.Assets()
 	LoadDotEnvs()
 
-	switch args[1] {
+	switch args[0] {
 	case "app":
 		handlers.DesktopInit(ipAddress, port)
 	case "version":
@@ -62,6 +62,8 @@ func CatchCLI(args []string) error {
 			return errors.New("end")
 		}
 	case "sass":
+		utils.InitLogs()
+		source.Assets()
 		err := source.CompileSASS(dir)
 		if err == nil {
 			return errors.New("end")
@@ -83,7 +85,7 @@ func CatchCLI(args []string) error {
 		}
 		return nil
 	case "test":
-		cmd := args[2]
+		cmd := args[1]
 		switch cmd {
 		case "plugins":
 			LoadPlugins(true)

cmd/cli_test.go

@@ -21,6 +21,12 @@ import (
 	"testing"
 )
 
+func TestRunSQLiteApp(t *testing.T) {
+	t.SkipNow()
+	run := CatchCLI([]string{"app"})
+	assert.Nil(t, run)
+}
+
 func TestConfirmVersion(t *testing.T) {
 	t.SkipNow()
 	assert.NotEmpty(t, VERSION)
@@ -54,52 +60,50 @@ func TestAssetsCommand(t *testing.T) {
 	t.Log(c.Stdout())
 	t.Log("Directory for Assets: ", dir)
 	assert.FileExists(t, dir+"/assets/robots.txt")
-	assert.FileExists(t, dir+"/assets/js/main.js")
 	assert.FileExists(t, dir+"/assets/scss/base.scss")
 }
 
 func TestVersionCLI(t *testing.T) {
-	run := CatchCLI([]string{"statup", "version"})
+	run := CatchCLI([]string{"version"})
 	assert.EqualError(t, run, "end")
 }
 
 func TestAssetsCLI(t *testing.T) {
-	t.SkipNow()
+	run := CatchCLI([]string{"assets"})
-	run := CatchCLI([]string{"statup", "assets"})
 	assert.EqualError(t, run, "end")
 	assert.FileExists(t, dir+"/assets/css/base.css")
 	assert.FileExists(t, dir+"/assets/scss/base.scss")
 }
 
 func TestSassCLI(t *testing.T) {
-	run := CatchCLI([]string{"statup", "sass"})
+	run := CatchCLI([]string{"sass"})
 	assert.EqualError(t, run, "end")
 	assert.FileExists(t, dir+"/assets/css/base.css")
 }
 
 func TestUpdateCLI(t *testing.T) {
 	t.SkipNow()
-	run := CatchCLI([]string{"statup", "update"})
+	run := CatchCLI([]string{"update"})
 	assert.EqualError(t, run, "end")
 }
 
 func TestTestPackageCLI(t *testing.T) {
-	run := CatchCLI([]string{"statup", "test", "plugins"})
+	run := CatchCLI([]string{"test", "plugins"})
 	assert.EqualError(t, run, "end")
 }
 
 func TestHelpCLI(t *testing.T) {
-	run := CatchCLI([]string{"statup", "help"})
+	run := CatchCLI([]string{"help"})
 	assert.EqualError(t, run, "end")
 }
 
 func TestRunOnceCLI(t *testing.T) {
 	t.SkipNow()
-	run := CatchCLI([]string{"statup", "run"})
+	run := CatchCLI([]string{"run"})
 	assert.Nil(t, run)
 }
 
 func TestEnvCLI(t *testing.T) {
-	run := CatchCLI([]string{"statup", "env"})
+	run := CatchCLI([]string{"env"})
 	assert.Error(t, run)
 }

cmd/main.go

@@ -56,9 +56,10 @@ func main() {
 	var err error
 	parseFlags()
 	utils.InitLogs()
+	args := flag.Args()
 
-	if len(os.Args) >= 2 {
+	if len(args) >= 1 {
-		err := CatchCLI(os.Args)
+		err := CatchCLI(args)
 		if err != nil {
 			if err.Error() == "end" {
 				os.Exit(0)
@@ -68,7 +69,6 @@ func main() {
 		}
 	}
 
-	utils.InitLogs()
 	source.Assets()
 	LoadDotEnvs()
 

cmd/main_debug.go

@@ -0,0 +1,100 @@
+// +build debug
+
+// Statup
+// Copyright (C) 2018.  Hunter Long and the project contributors
+// Written by Hunter Long <info@socialeck.com> and the project contributors
+//
+// https://github.com/hunterlong/statup
+//
+// The licenses for most software and other practical works are designed
+// to take away your freedom to share and change the works.  By contrast,
+// the GNU General Public License is intended to guarantee your freedom to
+// share and change all versions of a program--to make sure it remains free
+// software for all its users.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//
+//  Debug instance of Statup using pprof and debugcharts
+//
+//  go get -u github.com/google/pprof
+//  go get -v -u github.com/mkevac/debugcharts
+//
+//  debugcharts web interface is on http://localhost:9090
+//
+//  - pprof -http=localhost:6060 http://localhost:8080/debug/pprof/profile
+//  - pprof -http=localhost:6060 http://localhost:8080/debug/pprof/heap
+//  - pprof -http=localhost:6060 http://localhost:8080/debug/pprof/goroutine
+//  - pprof -http=localhost:6060 http://localhost:8080/debug/pprof/block
+//
+
+package main
+
+import (
+	"fmt"
+	gorillahandler "github.com/gorilla/handlers"
+	"github.com/hunterlong/statup/core"
+	"github.com/hunterlong/statup/handlers"
+	_ "github.com/mkevac/debugcharts"
+	"net/http"
+	"net/http/pprof"
+	"os"
+	"time"
+)
+
+func init() {
+	os.Setenv("GO_ENV", "test")
+	go func() {
+		time.Sleep(5 * time.Second)
+		r := handlers.ReturnRouter()
+		r.HandleFunc("/debug/pprof/", pprof.Index)
+		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
+		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+		r.Handle("/debug/pprof/block", pprof.Handler("block"))
+		handlers.UpdateRouter(r)
+		time.Sleep(5 * time.Second)
+		go ViewPagesLoop()
+	}()
+	go func() {
+		panic(http.ListenAndServe(":9090", gorillahandler.CompressHandler(http.DefaultServeMux)))
+	}()
+}
+
+func ViewPagesLoop() {
+	httpRequest("/")
+	httpRequest("/charts.js")
+	httpRequest("/css/base.css")
+	httpRequest("/css/bootstrap.min.css")
+	httpRequest("/js/main.js")
+	httpRequest("/js/jquery-3.3.1.min.js")
+	httpRequest("/login")
+	httpRequest("/dashboard")
+	httpRequest("/settings")
+	httpRequest("/users")
+	httpRequest("/users/1")
+	httpRequest("/services")
+	httpRequest("/help")
+	httpRequest("/logs")
+	httpRequest("/404pageishere")
+	for i := 1; i <= len(core.CoreApp.Services()); i++ {
+		httpRequest(fmt.Sprintf("/service/%v", i))
+	}
+	defer ViewPagesLoop()
+}
+
+func httpRequest(url string) {
+	domain := fmt.Sprintf("http://localhost:%v%v", port, url)
+	response, err := http.Get(domain)
+	if err != nil {
+		fmt.Printf("%s", err)
+		return
+	}
+	defer response.Body.Close()
+	time.Sleep(10 * time.Millisecond)
+}

core/checker.go

@@ -167,11 +167,12 @@ func (s *Service) checkHttp(record bool) *Service {
 		return s
 	}
 	defer response.Body.Close()
+
+	if s.Expected != "" {
 		contents, err := ioutil.ReadAll(response.Body)
 		if err != nil {
 			utils.Log(2, err)
 		}
-	if s.Expected != "" {
 		match, err := regexp.MatchString(s.Expected, string(contents))
 		if err != nil {
 			utils.Log(2, err)
@@ -186,14 +187,13 @@ func (s *Service) checkHttp(record bool) *Service {
 		}
 	}
 	if s.ExpectedStatus != response.StatusCode {
-		s.LastResponse = string(contents)
+		//s.LastResponse = string(contents)
 		s.LastStatusCode = response.StatusCode
 		if record {
 			RecordFailure(s, fmt.Sprintf("HTTP Status Code %v did not match %v", response.StatusCode, s.ExpectedStatus))
 		}
 		return s
 	}
-	s.LastResponse = string(contents)
 	s.LastStatusCode = response.StatusCode
 	s.Online = true
 	if record {

core/database.go

@@ -124,6 +124,23 @@ func DeleteAllSince(table string, date time.Time) {
 	}
 }
 
+func (c *DbConfig) Update() error {
+	var err error
+	config, err := os.Create(utils.Directory + "/config.yml")
+	if err != nil {
+		utils.Log(4, err)
+		return err
+	}
+	data, err := yaml.Marshal(c.DbConfig)
+	if err != nil {
+		utils.Log(3, err)
+		return err
+	}
+	config.WriteString(string(data))
+	config.Close()
+	return err
+}
+
 func (c *DbConfig) Save() error {
 	var err error
 	config, err := os.Create(utils.Directory + "/config.yml")
@@ -172,7 +189,8 @@ func (c *DbConfig) Save() error {
 		utils.Log(4, err)
 	}
 	CoreApp.DbConnection = c.DbConn
-
+	c.ApiKey = CoreApp.ApiKey
+	c.ApiSecret = CoreApp.ApiSecret
 	return err
 }
 

core/events.go

@@ -17,6 +17,7 @@ package core
 
 import (
 	"github.com/fatih/structs"
+	"github.com/hunterlong/statup/notifiers"
 	"github.com/hunterlong/statup/types"
 	"upper.io/db.v3/lib/sqlbuilder"
 )
@@ -31,16 +32,14 @@ func OnSuccess(s *Service) {
 	for _, p := range CoreApp.AllPlugins {
 		p.OnSuccess(structs.Map(s))
 	}
-	//notifiers.OnSuccess(s)
+	notifiers.OnSuccess(s.Service)
-	// TODO convert notifiers to correct type
 }
 
 func OnFailure(s *Service, f *types.Failure) {
 	for _, p := range CoreApp.AllPlugins {
 		p.OnFailure(structs.Map(s))
 	}
-	//notifiers.OnFailure(s)
+	notifiers.OnFailure(s.Service)
-	// TODO convert notifiers to correct type
 }
 
 func OnSettingsSaved(c *types.Core) {

core/failures.go

@@ -110,7 +110,11 @@ func (s *Service) TotalFailures24Hours() (uint64, error) {
 }
 
 func (f *Failure) ParseError() string {
-	err := strings.Contains(f.Issue, "operation timed out")
+	err := strings.Contains(f.Issue, "connection reset by peer")
+	if err {
+		return fmt.Sprintf("Connection Reset")
+	}
+	err = strings.Contains(f.Issue, "operation timed out")
 	if err {
 		return fmt.Sprintf("HTTP Request Timed Out")
 	}

core/services_test.go

@@ -313,3 +313,25 @@ func TestDNScheckService(t *testing.T) {
 	assert.Nil(t, err)
 	assert.NotZero(t, amount)
 }
+
+func TestGroupGraphData(t *testing.T) {
+	service := SelectService(1)
+	CoreApp.DbConnection = "mysql"
+	lastWeek := time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
+	out := GroupDataBy("services", service.Id, lastWeek, "hour")
+	t.Log(out)
+	assert.Contains(t, out, "SELECT CONCAT(date_format(created_at, '%Y-%m-%dT%H:%i:00Z'))")
+
+	CoreApp.DbConnection = "postgres"
+	lastWeek = time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
+	out = GroupDataBy("services", service.Id, lastWeek, "hour")
+	t.Log(out)
+	assert.Contains(t, out, "SELECT date_trunc('hour', created_at)")
+
+	CoreApp.DbConnection = "sqlite"
+	lastWeek = time.Now().Add(time.Hour*-(24*7) + time.Minute*0 + time.Second*0)
+	out = GroupDataBy("services", service.Id, lastWeek, "hour")
+	t.Log(out)
+	assert.Contains(t, out, "SELECT strftime('%Y-%m-%dT%H:%M:00Z'")
+
+}

handlers/benchmark_test.go

@@ -0,0 +1,47 @@
+// Statup
+// Copyright (C) 2018.  Hunter Long and the project contributors
+// Written by Hunter Long <info@socialeck.com> and the project contributors
+//
+// https://github.com/hunterlong/statup
+//
+// The licenses for most software and other practical works are designed
+// to take away your freedom to share and change the works.  By contrast,
+// the GNU General Public License is intended to guarantee your freedom to
+// share and change all versions of a program--to make sure it remains free
+// software for all its users.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package handlers
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func BenchmarkHandleIndex(b *testing.B) {
+	b.ReportAllocs()
+	r := request(b, "/")
+	for i := 0; i < b.N; i++ {
+		rw := httptest.NewRecorder()
+		IndexHandler(rw, r)
+	}
+}
+
+func BenchmarkServicesHandlerIndex(b *testing.B) {
+	r := request(b, "/")
+	for i := 0; i < b.N; i++ {
+		rw := httptest.NewRecorder()
+		ServicesHandler(rw, r)
+	}
+}
+
+func request(t testing.TB, url string) *http.Request {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}

handlers/handlers.go

@@ -51,8 +51,8 @@ func RunHTTPServer(ip string, port int) error {
 	router = Router()
 	httpServer = &http.Server{
 		Addr:         host,
-		WriteTimeout: time.Second * 15,
+		WriteTimeout: time.Second * 60,
-		ReadTimeout:  time.Second * 15,
+		ReadTimeout:  time.Second * 60,
 		IdleTimeout:  time.Second * 60,
 		Handler:      router,
 	}

handlers/handlers_test.go

@@ -16,6 +16,7 @@
 package handlers
 
 import (
+	"fmt"
 	"github.com/hunterlong/statup/core"
 	"github.com/hunterlong/statup/source"
 	"github.com/hunterlong/statup/utils"
@@ -143,7 +144,6 @@ func TestServiceChartHandler(t *testing.T) {
 	t.Log(body)
 	assert.Contains(t, body, "var ctx_1")
 	assert.Contains(t, body, "var ctx_3")
-	assert.Contains(t, body, "var ctx_4")
 	assert.Contains(t, body, "var ctx_5")
 }
 
@@ -476,7 +476,6 @@ func TestSaveAssetsHandler(t *testing.T) {
 	Router().ServeHTTP(rr, req)
 	assert.Equal(t, 200, rr.Code)
 	assert.FileExists(t, utils.Directory+"/assets/css/base.css")
-	assert.FileExists(t, utils.Directory+"/assets/js/main.js")
 	assert.DirExists(t, utils.Directory+"/assets")
 	assert.True(t, source.UsingAssets(dir))
 	assert.True(t, IsRouteAuthenticated(req))
@@ -608,3 +607,50 @@ func TestSaveSassHandler(t *testing.T) {
 	newBase := source.OpenAsset(utils.Directory, "css/base.css")
 	assert.Contains(t, newBase, ".test_design {")
 }
+
+func TestReorderServiceHandler(t *testing.T) {
+	data := `[{id: 1, order: 3},{id: 2, order: 2},{id: 3, order: 1}]"`
+	req, err := http.NewRequest("POST", "/services/reorder", strings.NewReader(data))
+	req.Header.Set("Content-Type", "application/json")
+	assert.Nil(t, err)
+	rr := httptest.NewRecorder()
+	Router().ServeHTTP(rr, req)
+	assert.Equal(t, 200, rr.Code)
+	assert.True(t, IsRouteAuthenticated(req))
+}
+
+func TestCreateBulkServices(t *testing.T) {
+	domains := []string{
+		"https://status.coinapp.io",
+		"https://demo.statup.io",
+		"https://golang.org",
+		"https://github.com/hunterlong",
+		"https://www.santamonica.com",
+		"https://www.oeschs-die-dritten.ch/en/",
+		"https://etherscan.io",
+		"https://www.youtube.com/watch?v=ipvEIZMMILA",
+		"https://www.youtube.com/watch?v=UdaYVxYF1Ok",
+		"https://www.youtube.com/watch?v=yydZbVoCbn0&t=870s",
+		"http://failingdomainsarenofunatall.com",
+	}
+	for k, d := range domains {
+		form := url.Values{}
+		form.Add("name", fmt.Sprintf("Test Service %v", k))
+		form.Add("domain", d)
+		form.Add("method", "GET")
+		form.Add("expected_status", "200")
+		form.Add("interval", fmt.Sprintf("%v", k+1))
+		form.Add("port", "")
+		form.Add("timeout", "30")
+		form.Add("check_type", "http")
+		form.Add("post_data", "")
+
+		req, err := http.NewRequest("POST", "/services", strings.NewReader(form.Encode()))
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		assert.Nil(t, err)
+		rr := httptest.NewRecorder()
+		Router().ServeHTTP(rr, req)
+		assert.Equal(t, 200, rr.Code)
+		assert.True(t, IsRouteAuthenticated(req))
+	}
+}

handlers/index.go

@@ -103,6 +103,10 @@ func DesktopInit(ip string, port int) {
 
 	core.LoadSampleData()
 
+	config.ApiKey = core.CoreApp.ApiKey
+	config.ApiSecret = core.CoreApp.ApiSecret
+	config.Update()
+
 	core.InitApp()
 	RunHTTPServer(ip, port)
 }

handlers/routes.go

@@ -37,17 +37,16 @@ func Router() *mux.Router {
 	if source.UsingAssets(dir) {
 		indexHandler := http.FileServer(http.Dir(dir + "/assets/"))
 		r.PathPrefix("/css/").Handler(http.StripPrefix("/css/", http.FileServer(http.Dir(dir+"/assets/css"))))
-		r.PathPrefix("/js/").Handler(http.StripPrefix("/js/", http.FileServer(http.Dir(dir+"/assets/js"))))
 		r.PathPrefix("/robots.txt").Handler(indexHandler)
 		r.PathPrefix("/favicon.ico").Handler(indexHandler)
 		r.PathPrefix("/statup.png").Handler(indexHandler)
 	} else {
 		r.PathPrefix("/css/").Handler(http.StripPrefix("/css/", http.FileServer(source.CssBox.HTTPBox())))
-		r.PathPrefix("/js/").Handler(http.StripPrefix("/js/", http.FileServer(source.JsBox.HTTPBox())))
 		r.PathPrefix("/robots.txt").Handler(http.FileServer(source.TmplBox.HTTPBox()))
 		r.PathPrefix("/favicon.ico").Handler(http.FileServer(source.TmplBox.HTTPBox()))
 		r.PathPrefix("/statup.png").Handler(http.FileServer(source.TmplBox.HTTPBox()))
 	}
+	r.PathPrefix("/js/").Handler(http.StripPrefix("/js/", http.FileServer(source.JsBox.HTTPBox())))
 	r.Handle("/charts.js", http.HandlerFunc(RenderServiceChartsHandler))
 	r.Handle("/setup", http.HandlerFunc(SetupHandler)).Methods("GET")
 	r.Handle("/setup", http.HandlerFunc(ProcessSetupHandler)).Methods("POST")
@@ -105,6 +104,15 @@ func Router() *mux.Router {
 	return r
 }
 
+func ReturnRouter() *mux.Router {
+	return router
+}
+
+func UpdateRouter(routes *mux.Router) {
+	router = routes
+	httpServer.Handler = router
+}
+
 func ResetRouter() {
 	router = Router()
 	httpServer.Handler = router

source/source.go

@@ -145,12 +145,12 @@ func CreateAllAssets(folder string) error {
 	CopyToPublic(ScssBox, folder+"/assets/scss", "mobile.scss")
 	CopyToPublic(CssBox, folder+"/assets/css", "bootstrap.min.css")
 	CopyToPublic(CssBox, folder+"/assets/css", "base.css")
-	CopyToPublic(JsBox, folder+"/assets/js", "bootstrap.min.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "bootstrap.min.js")
-	CopyToPublic(JsBox, folder+"/assets/js", "Chart.bundle.min.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "Chart.bundle.min.js")
-	CopyToPublic(JsBox, folder+"/assets/js", "jquery-3.3.1.min.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "jquery-3.3.1.min.js")
-	CopyToPublic(JsBox, folder+"/assets/js", "sortable.min.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "sortable.min.js")
-	CopyToPublic(JsBox, folder+"/assets/js", "main.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "main.js")
-	CopyToPublic(JsBox, folder+"/assets/js", "setup.js")
+	//CopyToPublic(JsBox, folder+"/assets/js", "setup.js")
 	CopyToPublic(TmplBox, folder+"/assets", "robots.txt")
 	CopyToPublic(TmplBox, folder+"/assets", "statup.png")
 	utils.Log(1, "Compiling CSS from SCSS style...")

types/types.go

@@ -85,6 +85,8 @@ type DbConfig struct {
 	DbPass      string `yaml:"password"`
 	DbData      string `yaml:"database"`
 	DbPort      int    `yaml:"port"`
+	ApiKey      string `yaml:"api_key"`
+	ApiSecret   string `yaml:"api_secret"`
 	Project     string `yaml:"-"`
 	Description string `yaml:"-"`
 	Domain      string `yaml:"-"`