From 413ec6c83aec2caec6736d39db14c7c7d71f141a Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Wed, 28 Oct 2015 11:38:03 -0700 Subject: [PATCH 1/2] Spark: Update example to use kubectl proxy rather than load balancer --- examples/spark/README.md | 45 ++++++++++----------------------- examples/spark/spark-webui.yaml | 1 - 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/examples/spark/README.md b/examples/spark/README.md index 3f1a5ae1ff..0ab609cd18 100644 --- a/examples/spark/README.md +++ b/examples/spark/README.md @@ -81,12 +81,7 @@ $ kubectl create -f examples/spark/spark-master-service.yaml services/spark-master ``` -Optionally, you can create a service for the Spark Master WebUI at this point as -well. If you are running on a cloud provider that supports it, this will create -an external load balancer and open a firewall to the Spark Master WebUI on the -cluster. **Note:** With the existing configuration, there is **ABSOLUTELY NO** -authentication on this WebUI. With slightly more work, it would be -straightforward to put an `nginx` proxy in front to password protect it. +You can then create a service for the Spark Master WebUI: ```console $ kubectl create -f examples/spark/spark-webui.yaml @@ -125,29 +120,16 @@ Spark Command: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -cp /opt/spark-1.5 15/10/27 21:25:07 INFO Master: I have been elected leader! New state: ALIVE ``` -If you created the Spark WebUI and waited sufficient time for the load balancer -to be create, the `spark-webui` service should look something like this: +After you know the master is running, you can use the (cluster +proxy)[../../docs/user-guide/accessing-the-cluster.md#using-kubectl-proxy] to +connect to the Spark WebUI: ```console -$ kubectl describe services/spark-webui -Name: spark-webui -Namespace: default -Labels: -Selector: component=spark-master -Type: LoadBalancer -IP: 10.0.152.249 -LoadBalancer Ingress: 104.197.147.190 -Port: 8080/TCP -NodePort: 31141/TCP -Endpoints: 10.244.1.12:8080 -Session Affinity: None -Events: [...] +kubectl proxy --port=8001 ``` -You should now be able to visit `http://104.197.147.190:8080` and see the Spark -Master UI. *Note:* After workers connect, this UI has links to worker Web -UIs. The worker UI links do not work (the links attempt to connect to cluster -IPs). +At which point the UI will be available at +http://localhost:8001/api/v1/proxy/namespaces/default/services/spark-webui/ ## Step Two: Start your Spark workers @@ -185,6 +167,11 @@ $ kubectl logs spark-master-controller-5u0q5 15/10/26 18:20:14 INFO Master: Registering worker 10.244.3.8:39926 with 2 cores, 6.3 GB RAM ``` +Assuming you still have the `kubectl proxy` running from the previous section, +you should now see the workers in the UI as well. *Note:* The UI will have links +to worker Web UIs. The worker UI links do not work (the links will attempt to +connect to cluster IPs, which Kubernetes won't proxy automatically). + ## Step Three: Start your Spark driver to launch jobs on your Spark cluster The Spark driver is used to launch jobs into Spark cluster. You can read more about it in @@ -241,18 +228,14 @@ information. ## tl;dr ```console -kubectl create -f examples/spark/spark-master-controller.yaml -kubectl create -f examples/spark/spark-master-service.yaml -kubectl create -f examples/spark/spark-webui.yaml -kubectl create -f examples/spark/spark-worker-controller.yaml -kubectl create -f examples/spark/spark-driver-controller.yaml +kubectl create -f examples/spark ``` After it's setup: ```console kubectl get pods # Make sure everything is running -kubectl get services spark-webui # Get the IP of the Spark WebUI +kubectl proxy --port=8001 # Start an application proxy, if you want to see the Spark WebUI kubectl get pods -lcomponent=spark-driver # Get the driver pod to interact with. ``` diff --git a/examples/spark/spark-webui.yaml b/examples/spark/spark-webui.yaml index 28e8abe435..421a3e4713 100644 --- a/examples/spark/spark-webui.yaml +++ b/examples/spark/spark-webui.yaml @@ -8,4 +8,3 @@ spec: targetPort: 8080 selector: component: spark-master - type: LoadBalancer From d24d2a21f98f20eeacc78da721023b43143d2524 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 29 Oct 2015 05:22:56 -0700 Subject: [PATCH 2/2] Spark: Add liveness probes for master/worker It turns out, the pods don't exit when the master or worker crash. Along the way: remove redundant metadata. --- examples/spark/spark-driver-controller.yaml | 2 -- examples/spark/spark-master-controller.yaml | 11 +++++++++-- examples/spark/spark-master-service.yaml | 2 -- examples/spark/spark-worker-controller.yaml | 12 +++++++++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/spark/spark-driver-controller.yaml b/examples/spark/spark-driver-controller.yaml index d62b3e2196..d0a3621a6f 100644 --- a/examples/spark/spark-driver-controller.yaml +++ b/examples/spark/spark-driver-controller.yaml @@ -2,8 +2,6 @@ kind: ReplicationController apiVersion: v1 metadata: name: spark-driver-controller - labels: - component: spark-driver spec: replicas: 1 selector: diff --git a/examples/spark/spark-master-controller.yaml b/examples/spark/spark-master-controller.yaml index b67c1424dc..d497b4183c 100644 --- a/examples/spark/spark-master-controller.yaml +++ b/examples/spark/spark-master-controller.yaml @@ -2,8 +2,6 @@ kind: ReplicationController apiVersion: v1 metadata: name: spark-master-controller - labels: - component: spark-master spec: replicas: 1 selector: @@ -19,6 +17,15 @@ spec: ports: - containerPort: 7077 - containerPort: 8080 + livenessProbe: + exec: + command: + - /opt/spark/sbin/spark-daemon.sh + - status + - org.apache.spark.deploy.master.Master + - '1' + initialDelaySeconds: 30 + timeoutSeconds: 1 resources: requests: cpu: 100m diff --git a/examples/spark/spark-master-service.yaml b/examples/spark/spark-master-service.yaml index ec51365b39..32d20a71eb 100644 --- a/examples/spark/spark-master-service.yaml +++ b/examples/spark/spark-master-service.yaml @@ -2,8 +2,6 @@ kind: Service apiVersion: v1 metadata: name: spark-master - labels: - component: spark-master-service spec: ports: - port: 7077 diff --git a/examples/spark/spark-worker-controller.yaml b/examples/spark/spark-worker-controller.yaml index 1fa59f04c2..8f30003bb6 100644 --- a/examples/spark/spark-worker-controller.yaml +++ b/examples/spark/spark-worker-controller.yaml @@ -2,8 +2,6 @@ kind: ReplicationController apiVersion: v1 metadata: name: spark-worker-controller - labels: - component: spark-worker spec: replicas: 3 selector: @@ -12,13 +10,21 @@ spec: metadata: labels: component: spark-worker - uses: spark-master spec: containers: - name: spark-worker image: gcr.io/google_containers/spark-worker:1.5.1_v1 ports: - containerPort: 8888 + livenessProbe: + exec: + command: + - /opt/spark/sbin/spark-daemon.sh + - status + - org.apache.spark.deploy.worker.Worker + - '1' + initialDelaySeconds: 30 + timeoutSeconds: 1 resources: requests: cpu: 100m