mirror of https://github.com/statping/statping
Merge pull request #806 from thatInfrastructureGuy/grpcCheckFixes
Enhance GRPC Monitoringpull/724/head^2
Hunter Long
GitHub
commit
fccc5595e6
|
|
@ -13,7 +13,7 @@
|
|||
<div class="form-group row">
|
||||
<label for="service_type" class="col-sm-4 col-form-label">{{ $t('service_type') }}</label>
|
||||
<div class="col-sm-8">
|
||||
<select v-model="service.type" class="form-control" id="service_type">
|
||||
<select v-model="service.type" @change="updateDefaultValues()" class="form-control" id="service_type">
|
||||
<option value="http">HTTP {{ $t('service') }}</option>
|
||||
<option value="tcp">TCP {{ $t('service') }}</option>
|
||||
<option value="udp">UDP {{ $t('service') }}</option>
|
||||
|
|
@ -155,8 +155,7 @@
|
|||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="service.type.match(/^(http)$/)" class="form-group row">
|
||||
<div v-if="service.type.match(/^(http|grpc)$/)" class="form-group row">
|
||||
<label class="col-12 col-md-4 col-form-label">{{ $t('verify_ssl') }}</label>
|
||||
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
|
||||
<span @click="service.verify_ssl = !!service.verify_ssl" class="switch float-left">
|
||||
|
|
@ -167,6 +166,33 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="service.type.match(/^(grpc)$/)" class="form-group row">
|
||||
<label class="col-12 col-md-4 col-form-label"><a href="https://github.com/grpc/grpc/blob/master/doc/health-checking.md#grpc-health-checking-protocol">GRPC Health Check</a></label>
|
||||
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
|
||||
<span @click="service.grpc_health_check = !!service.grpc_health_check" class="switch float-left">
|
||||
<input v-model="service.grpc_health_check" type="checkbox" name="grpc_health_check-option" class="switch" id="switch-grpc-health-check" v-bind:checked="service.grpc_health_check">
|
||||
<label for="switch-grpc-health-check" v-if="service.grpc_health_check">Check against GRPC health check endpoint.</label>
|
||||
<label for="switch-grpc-health-check" v-if="!service.grpc_health_check">Only checks if GRPC connection can be established.</label>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="service.grpc_health_check" class="form-group row">
|
||||
<label class="col-sm-4 col-form-label">Expected Response</label>
|
||||
<div class="col-sm-8">
|
||||
<textarea v-model="service.expected" class="form-control" rows="3" autocapitalize="none" spellcheck="false" placeholder='status:SERVING'></textarea>
|
||||
<small class="form-text text-muted">Check <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#pkg-variables">GPRC health check response codes</a> for more information.</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="service.grpc_health_check" class="form-group row">
|
||||
<label for="service_response_code" class="col-sm-4 col-form-label">Expected Status Code</label>
|
||||
<div class="col-sm-8">
|
||||
<input v-model="service.expected_status" type="number" name="expected_status" class="form-control" placeholder="1" id="service_response_code">
|
||||
<small class="form-text text-muted">A status code of 1 is success, or view all the <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#HealthCheckResponse_ServingStatus">GRPC Status Codes</a></small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="service.type.match(/^(tcp|http)$/)" class="form-group row">
|
||||
<label class="col-12 col-md-4 col-form-label">{{ $t('tls_cert') }}</label>
|
||||
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
|
||||
|
|
@ -276,6 +302,7 @@
|
|||
permalink: "",
|
||||
order: 1,
|
||||
verify_ssl: true,
|
||||
grpc_health_check: false,
|
||||
redirect: true,
|
||||
allow_notifications: true,
|
||||
notify_all_changes: true,
|
||||
|
|
@ -316,6 +343,21 @@
|
|||
this.service = this.in_service
|
||||
}
|
||||
this.use_tls = this.service.tls_cert !== ""
|
||||
},
|
||||
updateDefaultValues() {
|
||||
if (this.service.type === "grpc") {
|
||||
this.service.expected_status = 1
|
||||
this.service.expected = "status:SERVING"
|
||||
this.service.port = 50051
|
||||
this.service.verify_ssl = false
|
||||
this.service.method = ""
|
||||
} else {
|
||||
this.service.expected_status = 200
|
||||
this.service.expected = ""
|
||||
this.service.port = 80
|
||||
this.service.verify_ssl = true
|
||||
this.service.method = "GET"
|
||||
}
|
||||
},
|
||||
updatePermalink() {
|
||||
const a = 'àáâäæãåāăąçćčđďèéêëēėęěğǵḧîïíīįìłḿñńǹňôöòóœøōõőṕŕřßśšşșťțûüùúūǘůűųẃẍÿýžźż·/_,:;'
|
||||
|
|
|
|||
|
|
@ -2,11 +2,9 @@ package services
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/statping/statping/types/metrics"
|
||||
"google.golang.org/grpc"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -14,9 +12,15 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/statping/statping/types/metrics"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
|
||||
"github.com/statping/statping/types/failures"
|
||||
"github.com/statping/statping/types/hits"
|
||||
"github.com/statping/statping/utils"
|
||||
healthpb "google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
// checkServices will start the checking go routine for each service
|
||||
|
|
@ -115,6 +119,22 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
|
|||
timer := prometheus.NewTimer(metrics.ServiceTimer(s.Name))
|
||||
defer timer.ObserveDuration()
|
||||
|
||||
// Strip URL scheme if present. Eg: https:// , http://
|
||||
if strings.Contains(s.Domain, "://") {
|
||||
u, err := url.Parse(s.Domain)
|
||||
if err != nil {
|
||||
// Unable to parse.
|
||||
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Unable to parse URL: '%v'", s.Name, s.Domain))
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("Unable to parse GRPC domain %v, %v", s.Domain, err), "parse_domain")
|
||||
}
|
||||
}
|
||||
|
||||
// Set domain as hostname without port number.
|
||||
s.Domain = u.Hostname()
|
||||
}
|
||||
|
||||
// Calculate DNS check time
|
||||
dnsLookup, err := dnsCheck(s)
|
||||
if err != nil {
|
||||
if record {
|
||||
|
|
@ -122,6 +142,18 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
|
|||
}
|
||||
return s, err
|
||||
}
|
||||
|
||||
// Connect to grpc service without TLS certs.
|
||||
grpcOption := grpc.WithInsecure()
|
||||
|
||||
// Check if TLS is enabled
|
||||
// Upgrade GRPC connection if using TLS
|
||||
// Force to connect on HTTP2 with TLS. Needed when using a reverse proxy such as nginx.
|
||||
if s.VerifySSL.Bool {
|
||||
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
|
||||
grpcOption = grpc.WithTransportCredentials(h2creds)
|
||||
}
|
||||
|
||||
s.PingTime = dnsLookup
|
||||
t1 := utils.Now()
|
||||
domain := fmt.Sprintf("%v", s.Domain)
|
||||
|
|
@ -131,28 +163,70 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
|
|||
domain = fmt.Sprintf("[%v]:%v", s.Domain, s.Port)
|
||||
}
|
||||
}
|
||||
conn, err := grpc.Dial(domain, grpc.WithInsecure(), grpc.WithBlock())
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect: %v", err)
|
||||
}
|
||||
|
||||
// Context will cancel the request when timeout is exceeded.
|
||||
// Cancel the context when request is served within the timeout limit.
|
||||
timeout := time.Duration(s.Timeout) * time.Second
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
conn, err := grpc.DialContext(ctx, domain, grpcOption, grpc.WithBlock())
|
||||
if err != nil {
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("Dial Error %v", err), "connection")
|
||||
}
|
||||
return s, err
|
||||
}
|
||||
|
||||
if s.GrpcHealthCheck.Bool {
|
||||
// Create a new health check client
|
||||
c := healthpb.NewHealthClient(conn)
|
||||
in := &healthpb.HealthCheckRequest{}
|
||||
res, err := c.Check(ctx, in)
|
||||
if err != nil {
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("GRPC Error %v", err), "healthcheck")
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Record responses
|
||||
s.LastResponse = strings.TrimSpace(res.String())
|
||||
s.LastStatusCode = int(res.GetStatus())
|
||||
}
|
||||
|
||||
if err := conn.Close(); err != nil {
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("%v Socket Close Error %v", strings.ToUpper(s.Type), err), "close")
|
||||
}
|
||||
return s, err
|
||||
}
|
||||
|
||||
// Record latency
|
||||
s.Latency = utils.Now().Sub(t1).Microseconds()
|
||||
s.LastResponse = ""
|
||||
s.Online = true
|
||||
|
||||
if s.GrpcHealthCheck.Bool {
|
||||
if s.ExpectedStatus != s.LastStatusCode {
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("GRPC Service: '%s', Status Code: expected '%v', got '%v'", s.Name, s.ExpectedStatus, s.LastStatusCode), "response_code")
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
if s.Expected.String != s.LastResponse {
|
||||
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Response: expected '%v', got '%v'", s.Name, s.Expected.String, s.LastResponse))
|
||||
if record {
|
||||
RecordFailure(s, fmt.Sprintf("GRPC Response Body '%v' did not match '%v'", s.LastResponse, s.Expected.String), "response_body")
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
|
||||
if record {
|
||||
RecordSuccess(s)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,202 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/statping/statping/types/null"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/health"
|
||||
healthpb "google.golang.org/grpc/health/grpc_health_v1"
|
||||
)
|
||||
|
||||
// grpcServerDef is function type.
|
||||
// Consumed by Test data.
|
||||
type grpcServerDef func(int, bool) *grpc.Server
|
||||
|
||||
// Test Data: Simulates testing scenarios
|
||||
var testdata = []struct {
|
||||
grpcService grpcServerDef
|
||||
clientChecker *Service
|
||||
}{
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "GRPC Server with Health check",
|
||||
Domain: "localhost",
|
||||
Port: 50053,
|
||||
Expected: null.NewNullString("status:SERVING"),
|
||||
ExpectedStatus: 1,
|
||||
Type: "grpc",
|
||||
Timeout: 3,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check TLS endpoint on GRPC Server with TLS disabled",
|
||||
Domain: "localhost",
|
||||
Port: 50054,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(true),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check GRPC Server without Health check endpoint",
|
||||
Domain: "localhost",
|
||||
Port: 50055,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(50056, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check where no GRPC Server exists",
|
||||
Domain: "localhost",
|
||||
Port: 1000,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(50057, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check where no GRPC Server exists (Verify TLS)",
|
||||
Domain: "localhost",
|
||||
Port: 1000,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(true),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check GRPC Server with url",
|
||||
Domain: "http://localhost",
|
||||
Port: 50058,
|
||||
Expected: null.NewNullString("status:SERVING"),
|
||||
ExpectedStatus: 1,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Unparseable Url Error",
|
||||
Domain: "http://local//host",
|
||||
Port: 50059,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
GrpcHealthCheck: null.NewNullBool(true),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(50060, enableHealthCheck)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "Check GRPC on HTTP server",
|
||||
Domain: "https://google.com",
|
||||
Port: 443,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
},
|
||||
},
|
||||
{
|
||||
grpcService: func(port int, enableHealthCheck bool) *grpc.Server {
|
||||
return grpcServer(port, true)
|
||||
},
|
||||
clientChecker: &Service{
|
||||
Name: "GRPC HealthCheck where health check endpoint is not implemented",
|
||||
Domain: "http://localhost",
|
||||
Port: 50061,
|
||||
Expected: null.NewNullString(""),
|
||||
ExpectedStatus: 0,
|
||||
Type: "grpc",
|
||||
Timeout: 1,
|
||||
VerifySSL: null.NewNullBool(false),
|
||||
GrpcHealthCheck: null.NewNullBool(false),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// grpcServer creates grpc Service with optional parameters.
|
||||
func grpcServer(port int, enableHealthCheck bool) *grpc.Server {
|
||||
portString := strconv.Itoa(port)
|
||||
server := grpc.NewServer()
|
||||
lis, err := net.Listen("tcp", "localhost:"+portString)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
if enableHealthCheck {
|
||||
healthServer := health.NewServer()
|
||||
healthServer.SetServingStatus("Test GRPC Service", healthpb.HealthCheckResponse_SERVING)
|
||||
healthpb.RegisterHealthServer(server, healthServer)
|
||||
go server.Serve(lis)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
// TestCheckGrpc ranges over the testdata struct.
|
||||
// Examines checkGrpc() function
|
||||
func TestCheckGrpc(t *testing.T) {
|
||||
for _, testscenario := range testdata {
|
||||
v := testscenario
|
||||
t.Run(v.clientChecker.Name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := v.grpcService(v.clientChecker.Port, v.clientChecker.GrpcHealthCheck.Bool)
|
||||
defer server.Stop()
|
||||
v.clientChecker.CheckService(false)
|
||||
if v.clientChecker.LastStatusCode != v.clientChecker.ExpectedStatus || strings.TrimSpace(v.clientChecker.LastResponse) != v.clientChecker.Expected.String {
|
||||
t.Errorf("Expected message: '%v', Got message: '%v' , Expected Status: '%v', Got Status: '%v'", v.clientChecker.Expected.String, v.clientChecker.LastResponse, v.clientChecker.ExpectedStatus, v.clientChecker.LastStatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package services
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/statping/statping/types/checkins"
|
||||
"github.com/statping/statping/types/failures"
|
||||
"github.com/statping/statping/types/incidents"
|
||||
"github.com/statping/statping/types/messages"
|
||||
"github.com/statping/statping/types/null"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Service is the main struct for Services
|
||||
|
|
@ -24,6 +25,7 @@ type Service struct {
|
|||
Timeout int `gorm:"default:30;column:timeout" json:"timeout" scope:"user,admin" yaml:"timeout"`
|
||||
Order int `gorm:"default:0;column:order_id" json:"order_id" yaml:"order_id"`
|
||||
VerifySSL null.NullBool `gorm:"default:false;column:verify_ssl" json:"verify_ssl" scope:"user,admin" yaml:"verify_ssl"`
|
||||
GrpcHealthCheck null.NullBool `gorm:"default:false;column:grpc_health_check" json:"grpc_health_check" scope:"user,admin" yaml:"grpc_health_check"`
|
||||
Public null.NullBool `gorm:"default:true;column:public" json:"public" yaml:"public"`
|
||||
GroupId int `gorm:"default:0;column:group_id" json:"group_id" yaml:"group_id"`
|
||||
TLSCert null.NullString `gorm:"column:tls_cert" json:"tls_cert" scope:"user,admin" yaml:"tls_cert"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue