Merge pull request #806 from thatInfrastructureGuy/grpcCheckFixes

Enhance GRPC Monitoring
pull/724/head^2
Hunter Long 2020-10-01 09:58:05 -07:00 committed by GitHub
commit fccc5595e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 332 additions and 12 deletions

View File

@ -13,7 +13,7 @@
<div class="form-group row">
<label for="service_type" class="col-sm-4 col-form-label">{{ $t('service_type') }}</label>
<div class="col-sm-8">
<select v-model="service.type" class="form-control" id="service_type">
<select v-model="service.type" @change="updateDefaultValues()" class="form-control" id="service_type">
<option value="http">HTTP {{ $t('service') }}</option>
<option value="tcp">TCP {{ $t('service') }}</option>
<option value="udp">UDP {{ $t('service') }}</option>
@ -155,8 +155,7 @@
</span>
</div>
</div>
<div v-if="service.type.match(/^(http)$/)" class="form-group row">
<div v-if="service.type.match(/^(http|grpc)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label">{{ $t('verify_ssl') }}</label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
<span @click="service.verify_ssl = !!service.verify_ssl" class="switch float-left">
@ -167,6 +166,33 @@
</div>
</div>
<div v-if="service.type.match(/^(grpc)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label"><a href="https://github.com/grpc/grpc/blob/master/doc/health-checking.md#grpc-health-checking-protocol">GRPC Health Check</a></label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
<span @click="service.grpc_health_check = !!service.grpc_health_check" class="switch float-left">
<input v-model="service.grpc_health_check" type="checkbox" name="grpc_health_check-option" class="switch" id="switch-grpc-health-check" v-bind:checked="service.grpc_health_check">
<label for="switch-grpc-health-check" v-if="service.grpc_health_check">Check against GRPC health check endpoint.</label>
<label for="switch-grpc-health-check" v-if="!service.grpc_health_check">Only checks if GRPC connection can be established.</label>
</span>
</div>
</div>
<div v-if="service.grpc_health_check" class="form-group row">
<label class="col-sm-4 col-form-label">Expected Response</label>
<div class="col-sm-8">
<textarea v-model="service.expected" class="form-control" rows="3" autocapitalize="none" spellcheck="false" placeholder='status:SERVING'></textarea>
<small class="form-text text-muted">Check <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#pkg-variables">GPRC health check response codes</a> for more information.</small>
</div>
</div>
<div v-if="service.grpc_health_check" class="form-group row">
<label for="service_response_code" class="col-sm-4 col-form-label">Expected Status Code</label>
<div class="col-sm-8">
<input v-model="service.expected_status" type="number" name="expected_status" class="form-control" placeholder="1" id="service_response_code">
<small class="form-text text-muted">A status code of 1 is success, or view all the <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#HealthCheckResponse_ServingStatus">GRPC Status Codes</a></small>
</div>
</div>
<div v-if="service.type.match(/^(tcp|http)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label">{{ $t('tls_cert') }}</label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
@ -276,6 +302,7 @@
permalink: "",
order: 1,
verify_ssl: true,
grpc_health_check: false,
redirect: true,
allow_notifications: true,
notify_all_changes: true,
@ -316,6 +343,21 @@
this.service = this.in_service
}
this.use_tls = this.service.tls_cert !== ""
},
updateDefaultValues() {
if (this.service.type === "grpc") {
this.service.expected_status = 1
this.service.expected = "status:SERVING"
this.service.port = 50051
this.service.verify_ssl = false
this.service.method = ""
} else {
this.service.expected_status = 200
this.service.expected = ""
this.service.port = 80
this.service.verify_ssl = true
this.service.method = "GET"
}
},
updatePermalink() {
const a = 'àáâäæãåāăąçćčđďèéêëēėęěğǵḧîïíīįìłḿñńǹňôöòóœøōõőṕŕřßśšşșťțûüùúūǘůűųẃẍÿýžźż·/_,:;'

View File

@ -2,11 +2,9 @@ package services
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/statping/statping/types/metrics"
"google.golang.org/grpc"
"net"
"net/http"
"net/url"
@ -14,9 +12,15 @@ import (
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/statping/statping/types/metrics"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"github.com/statping/statping/types/failures"
"github.com/statping/statping/types/hits"
"github.com/statping/statping/utils"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
// checkServices will start the checking go routine for each service
@ -115,6 +119,22 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
timer := prometheus.NewTimer(metrics.ServiceTimer(s.Name))
defer timer.ObserveDuration()
// Strip URL scheme if present. Eg: https:// , http://
if strings.Contains(s.Domain, "://") {
u, err := url.Parse(s.Domain)
if err != nil {
// Unable to parse.
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Unable to parse URL: '%v'", s.Name, s.Domain))
if record {
RecordFailure(s, fmt.Sprintf("Unable to parse GRPC domain %v, %v", s.Domain, err), "parse_domain")
}
}
// Set domain as hostname without port number.
s.Domain = u.Hostname()
}
// Calculate DNS check time
dnsLookup, err := dnsCheck(s)
if err != nil {
if record {
@ -122,6 +142,18 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
}
return s, err
}
// Connect to grpc service without TLS certs.
grpcOption := grpc.WithInsecure()
// Check if TLS is enabled
// Upgrade GRPC connection if using TLS
// Force to connect on HTTP2 with TLS. Needed when using a reverse proxy such as nginx.
if s.VerifySSL.Bool {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
grpcOption = grpc.WithTransportCredentials(h2creds)
}
s.PingTime = dnsLookup
t1 := utils.Now()
domain := fmt.Sprintf("%v", s.Domain)
@ -131,28 +163,70 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
domain = fmt.Sprintf("[%v]:%v", s.Domain, s.Port)
}
}
conn, err := grpc.Dial(domain, grpc.WithInsecure(), grpc.WithBlock())
if err != nil {
log.Fatalf("did not connect: %v", err)
}
// Context will cancel the request when timeout is exceeded.
// Cancel the context when request is served within the timeout limit.
timeout := time.Duration(s.Timeout) * time.Second
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
conn, err := grpc.DialContext(ctx, domain, grpcOption, grpc.WithBlock())
if err != nil {
if record {
RecordFailure(s, fmt.Sprintf("Dial Error %v", err), "connection")
}
return s, err
}
if s.GrpcHealthCheck.Bool {
// Create a new health check client
c := healthpb.NewHealthClient(conn)
in := &healthpb.HealthCheckRequest{}
res, err := c.Check(ctx, in)
if err != nil {
if record {
RecordFailure(s, fmt.Sprintf("GRPC Error %v", err), "healthcheck")
}
return s, nil
}
// Record responses
s.LastResponse = strings.TrimSpace(res.String())
s.LastStatusCode = int(res.GetStatus())
}
if err := conn.Close(); err != nil {
if record {
RecordFailure(s, fmt.Sprintf("%v Socket Close Error %v", strings.ToUpper(s.Type), err), "close")
}
return s, err
}
// Record latency
s.Latency = utils.Now().Sub(t1).Microseconds()
s.LastResponse = ""
s.Online = true
if s.GrpcHealthCheck.Bool {
if s.ExpectedStatus != s.LastStatusCode {
if record {
RecordFailure(s, fmt.Sprintf("GRPC Service: '%s', Status Code: expected '%v', got '%v'", s.Name, s.ExpectedStatus, s.LastStatusCode), "response_code")
}
return s, nil
}
if s.Expected.String != s.LastResponse {
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Response: expected '%v', got '%v'", s.Name, s.Expected.String, s.LastResponse))
if record {
RecordFailure(s, fmt.Sprintf("GRPC Response Body '%v' did not match '%v'", s.LastResponse, s.Expected.String), "response_body")
}
return s, nil
}
}
if record {
RecordSuccess(s)
}
return s, nil
}

View File

@ -0,0 +1,202 @@
package services
import (
"fmt"
"net"
"strconv"
"strings"
"testing"
"github.com/statping/statping/types/null"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
// grpcServerDef is function type.
// Consumed by Test data.
type grpcServerDef func(int, bool) *grpc.Server
// Test Data: Simulates testing scenarios
var testdata = []struct {
grpcService grpcServerDef
clientChecker *Service
}{
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, enableHealthCheck)
},
clientChecker: &Service{
Name: "GRPC Server with Health check",
Domain: "localhost",
Port: 50053,
Expected: null.NewNullString("status:SERVING"),
ExpectedStatus: 1,
Type: "grpc",
Timeout: 3,
VerifySSL: null.NewNullBool(false),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check TLS endpoint on GRPC Server with TLS disabled",
Domain: "localhost",
Port: 50054,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(true),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check GRPC Server without Health check endpoint",
Domain: "localhost",
Port: 50055,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(50056, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check where no GRPC Server exists",
Domain: "localhost",
Port: 1000,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(50057, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check where no GRPC Server exists (Verify TLS)",
Domain: "localhost",
Port: 1000,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(true),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check GRPC Server with url",
Domain: "http://localhost",
Port: 50058,
Expected: null.NewNullString("status:SERVING"),
ExpectedStatus: 1,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, enableHealthCheck)
},
clientChecker: &Service{
Name: "Unparseable Url Error",
Domain: "http://local//host",
Port: 50059,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
GrpcHealthCheck: null.NewNullBool(true),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(50060, enableHealthCheck)
},
clientChecker: &Service{
Name: "Check GRPC on HTTP server",
Domain: "https://google.com",
Port: 443,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
},
},
{
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
return grpcServer(port, true)
},
clientChecker: &Service{
Name: "GRPC HealthCheck where health check endpoint is not implemented",
Domain: "http://localhost",
Port: 50061,
Expected: null.NewNullString(""),
ExpectedStatus: 0,
Type: "grpc",
Timeout: 1,
VerifySSL: null.NewNullBool(false),
GrpcHealthCheck: null.NewNullBool(false),
},
},
}
// grpcServer creates grpc Service with optional parameters.
func grpcServer(port int, enableHealthCheck bool) *grpc.Server {
portString := strconv.Itoa(port)
server := grpc.NewServer()
lis, err := net.Listen("tcp", "localhost:"+portString)
if err != nil {
fmt.Println(err)
}
if enableHealthCheck {
healthServer := health.NewServer()
healthServer.SetServingStatus("Test GRPC Service", healthpb.HealthCheckResponse_SERVING)
healthpb.RegisterHealthServer(server, healthServer)
go server.Serve(lis)
}
return server
}
// TestCheckGrpc ranges over the testdata struct.
// Examines checkGrpc() function
func TestCheckGrpc(t *testing.T) {
for _, testscenario := range testdata {
v := testscenario
t.Run(v.clientChecker.Name, func(t *testing.T) {
t.Parallel()
server := v.grpcService(v.clientChecker.Port, v.clientChecker.GrpcHealthCheck.Bool)
defer server.Stop()
v.clientChecker.CheckService(false)
if v.clientChecker.LastStatusCode != v.clientChecker.ExpectedStatus || strings.TrimSpace(v.clientChecker.LastResponse) != v.clientChecker.Expected.String {
t.Errorf("Expected message: '%v', Got message: '%v' , Expected Status: '%v', Got Status: '%v'", v.clientChecker.Expected.String, v.clientChecker.LastResponse, v.clientChecker.ExpectedStatus, v.clientChecker.LastStatusCode)
}
})
}
}

View File

@ -1,12 +1,13 @@
package services
import (
"time"
"github.com/statping/statping/types/checkins"
"github.com/statping/statping/types/failures"
"github.com/statping/statping/types/incidents"
"github.com/statping/statping/types/messages"
"github.com/statping/statping/types/null"
"time"
)
// Service is the main struct for Services
@ -24,6 +25,7 @@ type Service struct {
Timeout int `gorm:"default:30;column:timeout" json:"timeout" scope:"user,admin" yaml:"timeout"`
Order int `gorm:"default:0;column:order_id" json:"order_id" yaml:"order_id"`
VerifySSL null.NullBool `gorm:"default:false;column:verify_ssl" json:"verify_ssl" scope:"user,admin" yaml:"verify_ssl"`
GrpcHealthCheck null.NullBool `gorm:"default:false;column:grpc_health_check" json:"grpc_health_check" scope:"user,admin" yaml:"grpc_health_check"`
Public null.NullBool `gorm:"default:true;column:public" json:"public" yaml:"public"`
GroupId int `gorm:"default:0;column:group_id" json:"group_id" yaml:"group_id"`
TLSCert null.NullString `gorm:"column:tls_cert" json:"tls_cert" scope:"user,admin" yaml:"tls_cert"`