github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Retry Elasticsearch logging health check

pull/6/head
Satnam Singh 2015-06-23 09:36:56 -07:00
parent cb9f7d17f5
commit 52461b772a
1 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
test/e2e/es_cluster_logging.go
View File

@ -63,13 +63,18 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
return return
} }
// graceTime is how long to keep retrying requests for status information.
const graceTime = 2 * time.Minute
// ingestionTimeout is how long to keep retrying to wait for all the
// logs to be ingested.
const ingestionTimeout = 3 * time.Minute
// Check for the existence of the Elasticsearch service. // Check for the existence of the Elasticsearch service.
By("Checking the Elasticsearch service exists.") By("Checking the Elasticsearch service exists.")
s := f.Client.Services(api.NamespaceDefault) s := f.Client.Services(api.NamespaceDefault)
// Make a few attempts to connect. This makes the test robust against // Make a few attempts to connect. This makes the test robust against
// being run as the first e2e test just after the e2e cluster has been created. // being run as the first e2e test just after the e2e cluster has been created.
var err error var err error
const graceTime = 10 * time.Minute
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) { for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
if _, err = s.Get("elasticsearch-logging"); err == nil { if _, err = s.Get("elasticsearch-logging"); err == nil {
break break
@ -139,7 +144,9 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
// Now assume we really are talking to an Elasticsearch instance. // Now assume we really are talking to an Elasticsearch instance.
// Check the cluster health. // Check the cluster health.
By("Checking health of Elasticsearch service.") By("Checking health of Elasticsearch service.")
body, err := f.Client.Get(). var body []byte
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
body, err = f.Client.Get().
Namespace(api.NamespaceDefault). Namespace(api.NamespaceDefault).
Prefix("proxy"). Prefix("proxy").
Resource("services"). Resource("services").
@ -147,6 +154,10 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
Suffix("_cluster/health"). Suffix("_cluster/health").
Param("health", "pretty"). Param("health", "pretty").
DoRaw() DoRaw()
if err == nil {
break
}
}
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
health, err := bodyToJSON(body) health, err := bodyToJSON(body)
@ -244,7 +255,7 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
By("Checking all the log lines were ingested into Elasticsearch") By("Checking all the log lines were ingested into Elasticsearch")
missing := 0 missing := 0
expected := nodeCount * countTo expected := nodeCount * countTo
for start := time.Now(); time.Since(start) < graceTime; time.Sleep(10 * time.Second) { for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(10 * time.Second) {
// Debugging code to report the status of the elasticsearch logging endpoints. // Debugging code to report the status of the elasticsearch logging endpoints.
esPods, err := f.Client.Pods(api.NamespaceDefault).List(labels.Set{esKey: esValue}.AsSelector(), fields.Everything()) esPods, err := f.Client.Pods(api.NamespaceDefault).List(labels.Set{esKey: esValue}.AsSelector(), fields.Everything())