From ab4fc22878433c7ae86909d1e05e68aefdda1c2a Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 9 Jan 2020 12:14:41 +0000 Subject: [PATCH 1/5] submit applications depend on the service type --- bitnami/spark/templates/NOTES.txt | 42 +++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/bitnami/spark/templates/NOTES.txt b/bitnami/spark/templates/NOTES.txt index 885619f1b..ad1d2e8a3 100644 --- a/bitnami/spark/templates/NOTES.txt +++ b/bitnami/spark/templates/NOTES.txt @@ -1,21 +1,29 @@ 1. Get the Spark master WebUI URL by running these commands: {{- if .Values.ingress.enabled }} + export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ include "spark.fullname" . }}-ingress -o jsonpath='{.spec.rules[0].host}') echo "Spark-master URL: http://$HOSTNAME/" -{{- else }} + +{{- else -}} {{- if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "spark.master.service.name" . }}) + + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[?(@.name=='http')].nodePort}" services {{ include "spark.master.service.name" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT + {{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "spark.master.service.name" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') echo http://$SERVICE_IP:{{ .Values.service.webPort }} + {{- else if contains "ClusterIP" .Values.service.type }} + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "spark.master.service.name" . }} {{ default "80" .Values.service.webPort }}:{{ default "80" .Values.service.webPort }} echo "Visit http://127.0.0.1:{{ .Values.service.webPort }} to use your application" + {{- end }} {{- end }} @@ -26,17 +34,37 @@ First, obtain the master IP, to do that the service type must be NodePort or LoadBalancer. Run the following command to obtain the master IP and submit your application: -{{ if or (eq "NodePort" .Values.service.type) (eq "LoadBalancer" .Values.service.type) }} - export MASTER_IP=$(kubectl get services | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') - docker run -ti bitnami/spark spark-submit --master spark://$MASTER_IP:{{ .Values.service.clusterPort }} \ +{{- if eq "NodePort" .Values.service.type }} + + export SUBMIT_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[?(@.name=='cluster')].nodePort}" services {{ include "spark.master.service.name" . }}) + export SUBMIT_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + + docker run -ti bitnami/spark spark-submit --master spark://$SUBMIT_IP:$SUBMIT_PORT \ + --class org.apache.spark.examples.SparkPi \ + --deploy-mode cluster \ + examples/jars/spark-examples_2.11-2.4.4.jar 1000 + +{{- else if eq "LoadBalancer" .Values.service.type }} + + export SUBMIT_IP=$(kubectl get services | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') + + docker run -ti bitnami/spark spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.clusterPort }} \ --deploy-mode cluster \ --class org.apache.spark.examples.SparkPi \ examples/jars/spark-examples_2.11-2.4.4.jar 1000 + {{- else }} - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "spark.master.service.name" . }} {{ default "7077" .Values.service.clusterPort }}:{{ default "7077" .Values.service.clusterPort }} - spark-submit --master spark://127.0.0.1:{{ .Values.master.clusterPort }} --deploy-mode cluster /path/to/application 1000 + + export SUBMIT_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} | awk '/{{ include "spark.master.service.name" . }}/ { print $3 }') + + kubectl exec -ti {{ include "spark.fullname" . }}-worker-0 -- spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.clusterPort }} \ + --class org.apache.spark.examples.SparkPi \ + examples/jars/spark-examples_2.11-2.4.4.jar 5 + {{- end }} +** IMPORTANT: When submit an application from outside the cluster service type should be set to the NodePort or LoadBalancer. ** + ** IMPORTANT: When submit an application the --master parameter should be set to the service IP, if not, the application will not resolve the master. ** ** Please be patient while the chart is being deployed ** From 264636c8e4197cca71f6f6c2bdb9cd9ac164b66d Mon Sep 17 00:00:00 2001 From: darteaga Date: Thu, 9 Jan 2020 12:15:59 +0000 Subject: [PATCH 2/5] bump version of the chart --- bitnami/spark/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/spark/Chart.yaml b/bitnami/spark/Chart.yaml index 3125e557b..9255687c2 100644 --- a/bitnami/spark/Chart.yaml +++ b/bitnami/spark/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -version: 1.2.1 +version: 1.2.2 appVersion: 2.4.4 description: Spark is a fast and general-purpose cluster computing system. name: spark From 8a464909843c172e1ccbb76a3d4e5e3a8aaabae9 Mon Sep 17 00:00:00 2001 From: darteaga Date: Fri, 10 Jan 2020 11:55:03 +0000 Subject: [PATCH 3/5] PR requests --- bitnami/spark/templates/NOTES.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bitnami/spark/templates/NOTES.txt b/bitnami/spark/templates/NOTES.txt index ad1d2e8a3..e8f06239e 100644 --- a/bitnami/spark/templates/NOTES.txt +++ b/bitnami/spark/templates/NOTES.txt @@ -30,7 +30,7 @@ 2. Submit an application to the cluster: To submit an application to the cluster the spark-submit script must be used. That script can be - obtained at https://github.com/apache/spark/tree/master/bin. Also you can use bitnami/spark docker image. + obtained at https://github.com/apache/spark/tree/master/bin. Also you can use kubectl run. First, obtain the master IP, to do that the service type must be NodePort or LoadBalancer. Run the following command to obtain the master IP and submit your application: @@ -39,16 +39,20 @@ export SUBMIT_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[?(@.name=='cluster')].nodePort}" services {{ include "spark.master.service.name" . }}) export SUBMIT_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - docker run -ti bitnami/spark spark-submit --master spark://$SUBMIT_IP:$SUBMIT_PORT \ - --class org.apache.spark.examples.SparkPi \ - --deploy-mode cluster \ - examples/jars/spark-examples_2.11-2.4.4.jar 1000 + kubectl run --namespace {{ .Release.Namespace }} {{ template "spark.fullname" . }}-client --rm --tty -i --restart='Never' \ + --image {{ template "spark.image" . }} \ + -- spark-submit --master spark://$SUBMIT_IP:$SUBMIT_PORT \ + --class org.apache.spark.examples.SparkPi \ + --deploy-mode cluster \ + examples/jars/spark-examples_2.11-2.4.4.jar 1000 {{- else if eq "LoadBalancer" .Values.service.type }} export SUBMIT_IP=$(kubectl get services | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') - docker run -ti bitnami/spark spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.clusterPort }} \ + kubectl run --namespace {{ .Release.Namespace }} {{ template "spark.fullname" . }}-client --rm --tty -i --restart='Never' \ + --image {{ template "spark.image" . }} \ + -- spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.clusterPort }} \ --deploy-mode cluster \ --class org.apache.spark.examples.SparkPi \ examples/jars/spark-examples_2.11-2.4.4.jar 1000 From e9728b922382b7c2c061a23baf1404dbb6abd661 Mon Sep 17 00:00:00 2001 From: darteaga Date: Tue, 14 Jan 2020 15:07:31 +0000 Subject: [PATCH 4/5] remove awk usage from NOTE.txt --- bitnami/spark/templates/NOTES.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bitnami/spark/templates/NOTES.txt b/bitnami/spark/templates/NOTES.txt index e8f06239e..202e70538 100644 --- a/bitnami/spark/templates/NOTES.txt +++ b/bitnami/spark/templates/NOTES.txt @@ -16,7 +16,7 @@ NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "spark.master.service.name" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') + export SERVICE_IP=$(kubectl get --namespace {{ .Release.Namespace }} svc {{ include "spark.master.service.name" . }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }") echo http://$SERVICE_IP:{{ .Values.service.webPort }} {{- else if contains "ClusterIP" .Values.service.type }} @@ -48,7 +48,7 @@ {{- else if eq "LoadBalancer" .Values.service.type }} - export SUBMIT_IP=$(kubectl get services | awk '/{{ include "spark.master.service.name" . }}/ { print $4 }') + export SUBMIT_IP=$(kubectl get --namespace {{ .Release.Namespace }} svc {{ include "spark.master.service.name" . }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }") kubectl run --namespace {{ .Release.Namespace }} {{ template "spark.fullname" . }}-client --rm --tty -i --restart='Never' \ --image {{ template "spark.image" . }} \ @@ -59,9 +59,7 @@ {{- else }} - export SUBMIT_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} | awk '/{{ include "spark.master.service.name" . }}/ { print $3 }') - - kubectl exec -ti {{ include "spark.fullname" . }}-worker-0 -- spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.clusterPort }} \ + kubectl exec -ti {{ include "spark.fullname" . }}-worker-0 -- spark-submit --master spark://{{ include "spark.master.service.name" . }}:{{ .Values.service.clusterPort }} \ --class org.apache.spark.examples.SparkPi \ examples/jars/spark-examples_2.11-2.4.4.jar 5 From 8892589503a45830dec0350fccd16e9592480be3 Mon Sep 17 00:00:00 2001 From: darteaga Date: Wed, 15 Jan 2020 15:15:26 +0000 Subject: [PATCH 5/5] make instruction more accurate on cluster ip scenario --- bitnami/spark/templates/NOTES.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bitnami/spark/templates/NOTES.txt b/bitnami/spark/templates/NOTES.txt index 202e70538..7a5abf0ee 100644 --- a/bitnami/spark/templates/NOTES.txt +++ b/bitnami/spark/templates/NOTES.txt @@ -32,7 +32,10 @@ To submit an application to the cluster the spark-submit script must be used. That script can be obtained at https://github.com/apache/spark/tree/master/bin. Also you can use kubectl run. - First, obtain the master IP, to do that the service type must be NodePort or LoadBalancer. Run the following command to obtain the master IP and submit your application: +{{- if or (eq "NodePort" .Values.service.type) (eq "LoadBalancer" .Values.service.type) }} + + Run the commands below to obtain the master IP and submit your application. +{{- end -}} {{- if eq "NodePort" .Values.service.type }} @@ -63,10 +66,10 @@ --class org.apache.spark.examples.SparkPi \ examples/jars/spark-examples_2.11-2.4.4.jar 5 -{{- end }} - ** IMPORTANT: When submit an application from outside the cluster service type should be set to the NodePort or LoadBalancer. ** +{{- end }} + ** IMPORTANT: When submit an application the --master parameter should be set to the service IP, if not, the application will not resolve the master. ** ** Please be patient while the chart is being deployed **