From 448c8a66f195d56e3b8f7351927774548dfcf6eb Mon Sep 17 00:00:00 2001 From: miguelaeh Date: Mon, 17 Jun 2019 18:16:34 +0200 Subject: [PATCH] Add Spark Helm Chart Signed-off-by: Miguel Angel Cabrera Minagorri (miguelaeh) --- README.md | 1 + bitnami/spark/.helmignore | 22 ++ bitnami/spark/Chart.yaml | 15 + bitnami/spark/README.md | 276 ++++++++++++++++++ bitnami/spark/templates/NOTES.txt | 42 +++ bitnami/spark/templates/_helpers.tpl | 100 +++++++ bitnami/spark/templates/headless-svc.yaml | 15 + bitnami/spark/templates/hpa-worker.yaml | 23 ++ bitnami/spark/templates/ingress.yaml | 30 ++ bitnami/spark/templates/secret.yaml | 21 ++ .../spark/templates/statefulset-master.yaml | 167 +++++++++++ .../spark/templates/statefulset-worker.yaml | 188 ++++++++++++ bitnami/spark/templates/svc-master.yaml | 32 ++ bitnami/spark/values-production.yaml | 259 ++++++++++++++++ bitnami/spark/values.yaml | 259 ++++++++++++++++ 15 files changed, 1450 insertions(+) create mode 100644 bitnami/spark/.helmignore create mode 100644 bitnami/spark/Chart.yaml create mode 100644 bitnami/spark/README.md create mode 100644 bitnami/spark/templates/NOTES.txt create mode 100644 bitnami/spark/templates/_helpers.tpl create mode 100644 bitnami/spark/templates/headless-svc.yaml create mode 100644 bitnami/spark/templates/hpa-worker.yaml create mode 100644 bitnami/spark/templates/ingress.yaml create mode 100644 bitnami/spark/templates/secret.yaml create mode 100644 bitnami/spark/templates/statefulset-master.yaml create mode 100644 bitnami/spark/templates/statefulset-worker.yaml create mode 100644 bitnami/spark/templates/svc-master.yaml create mode 100644 bitnami/spark/values-production.yaml create mode 100644 bitnami/spark/values.yaml diff --git a/README.md b/README.md index 67f566a87..2a7511f11 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ $ helm search bitnami - [Apache](https://github.com/bitnami/charts/tree/master/bitnami/apache) - [Apache Airflow](https://github.com/bitnami/charts/tree/master/bitnami/airflow) +- [Apache Spark](https://github.com/bitnami/charts/tree/master/bitnami/spark) - [Cassandra](https://github.com/bitnami/charts/tree/master/bitnami/cassandra) - [Elasticsearch](https://github.com/bitnami/charts/tree/master/bitnami/elasticsearch) - [ExternalDNS](https://github.com/bitnami/charts/tree/master/bitnami/external-dns) diff --git a/bitnami/spark/.helmignore b/bitnami/spark/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/bitnami/spark/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/bitnami/spark/Chart.yaml b/bitnami/spark/Chart.yaml new file mode 100644 index 000000000..acaff044b --- /dev/null +++ b/bitnami/spark/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +appVersion: 2.4.3 +description: Spark is a fast and general-purpose cluster computing system. +name: spark +version: 0.0.1 +icon: https://bitnami.com/assets/stacks/spark/img/spark-stack-220x234-1ea65541e9e427ca5b93300e61d4778576c7f679ae02addb86a2641b0ca70476.png +home: https://spark.apache.org/ +sources: +- https://github.com/bitnami/bitnami-docker-spark +maintainers: +- name: Bitnami + email: containers@bitnami.com +keywords: +- apache +- spark diff --git a/bitnami/spark/README.md b/bitnami/spark/README.md new file mode 100644 index 000000000..8f4a08210 --- /dev/null +++ b/bitnami/spark/README.md @@ -0,0 +1,276 @@ +# Apache Spark + +[Apache Spark](https://spark.apache.org/) is a high-performance engine for large-scale computing tasks, such as data processing, machine learning and real-time data streaming. It includes APIs for Java, Python, Scala and R. + +## TL;DR; + +```console +$ helm install bitnami/spark +``` + +## Introduction + +This chart bootstraps a [spark](https://github.com/bitnami/bitnami-docker-spark) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install --name my-release bitnami/spark +``` + +The command deploys spark on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` statefulset: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. Use the option `--purge` to delete all persistent volumes too. + +## Configuration + +The following tables lists the configurable parameters of the spark chart and their default values. + +| Parameter | Description | Default | +| ------------------------------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.registry` | spark image registry | `docker.io` | +| `image.repository` | spark Image name | `bitnami/spark` | +| `image.tag` | spark Image tag | `{TAG_NAME}` | +| `image.pullPolicy` | spark image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `master.debug` | Specify if debug values should be set on the master | `false` | +| `master.webPort` | Specify the port where the web interface will listen on the master | `8080` | +| `master.clusterPort` | Specify the port where the master listens to communicate with workers | `7077` | +| `master.daemonMemoryLimit` | Set the memory limit for the master daemon | No default | +| `master.configOptions` | Optional configuration if the form `-Dx=y` | No default | +| `master.securityContext.enabled` | Enable security context | `true` | +| `master.securityContext.fsGroup` | Group ID for the container | `1001` | +| `master.securityContext.runAsUser` | User ID for the container | `1001` | +| `master.nodeSelector` | Node labels for pod assignment. Used ss a template from the values. | `{}` | +| `master.tolerations` | Toleration labels for pod assignment | `[]` | +| `master.affinity` | Affinity and AntiAffinity rules for pod assignment | `{}` | +| `master.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 10 | +| `master.livenessProbe.periodSeconds` | How often to perform the probe | 10 | +| `master.livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `master.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 2 | +| `master.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `master.readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 5 | +| `master.readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `master.readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `master.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `master.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `worker.debug` | Specify if debug values should be set on workers | `false` | +| `worker.webPort` | Specify the port where the web interface will listen on the worker | `8080` | +| `worker.clusterPort` | Specify the port where the worker listens to communicate with the master | `7077` | +| `worker.daemonMemoryLimit` | Set the memory limit for the worker daemon | No default | +| `worker.memoryLimit` | Set the maximum memory the worker is allowed to use | No default | +| `worker.coreLimit` | Se the maximum number of cores that the worker can use | No default | +| `worker.dir` | Set a custom working directory for the application | No default | +| `worker.javaOptions` | Set options for the JVM in the form `-Dx=y` | No default | +| `worker.configOptions` | Set extra options to configure the worker in the form `-Dx=y` | No default | +| `worker.replicaCount` | Set the number of workers | `2` | +| `worker.enableAutoscaling` | Enable autoscaling depending on CPU | `false` | +| `worker.replicasMax` | Maximum number of workers when using autoscaling | `false` | +| `worker.replicaCount` | Number of worker replicas | `1` | +| `worker.securityContext.enabled` | Enable security context | `true` | +| `worker.securityContext.fsGroup` | Group ID for the container | `1001` | +| `worker.securityContext.runAsUser` | User ID for the container | `1001` | +| `worker.nodeSelector` | Node labels for pod assignment. Used as a template from the values. | `{}` | +| `worker.tolerations` | Toleration labels for pod assignment | `[]` | +| `worker.affinity` | Affinity and AntiAffinity rules for pod assignment | `{}` | +| `worker.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | +| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 10 | +| `worker.livenessProbe.periodSeconds` | How often to perform the probe | 10 | +| `worker.livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `worker.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 2 | +| `worker.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` | +| `worker.readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 5 | +| `worker.readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `worker.readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `worker.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `worker.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `security.passwordsSecretName` | Secret to use when using security configuration to set custom passwords | No default | +| `security.rpc.authenticationEnabled` | Enable the RPC authentication | `no` | +| `security.rpc.encryptionEnabled` | Enable the encryption for RPC | `no` | +| `security.storageEncryptionEnabled` | Enable the encryption of the storage | `no` | +| `security.ssl.enabled` | Enable the SSL configuration | `no` | +| `security.ssl.needClientAuth` | Enable the client authentication | `no` | +| `security.ssl.protocol` | Set the SSL protocol | `TLSv1.2` | +| `security.certificatesSecretName` | Set the name of the secret that contains the certificates | No default | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | spark client port | `2379` | +| `service.nodePort` | Port to bind to for NodePort service type (client port) | `nil` | +| `service.annotations` | Annotations for spark service | {} | +| `service.loadBalancerIP` | loadBalancerIP if spark service type is `LoadBalancer` | `nil` | +| `ingress.enabled` | Enable the use of the ingress controller to access the web UI | `false` | +| `ingress.hosts` | Add hosts to the ingress controller with name and path | `name: spark.local`, `path: /` | + + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name my-release \ + --set master.webPort=8081 bitnami/spark +``` + +The above command sets the spark master web port to `8081`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install --name my-release -f values.yaml bitnami/spark +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +### Using custom configuration + +To use a custom configuration a ConfigMap should be created with the `spark-env.sh` file inside the ConfigMap. The ConfigMap name must be provided at deployment time, to set the configuration on the master use: +```bash +$ helm install bitnami/spark --set master.configurationConfigMap=configMapName +``` +To set the configuration on the worker use: +```bash +$ helm install bitnami/spark --set worker.configurationConfigMap=configMapName +``` +It can be set both at the same time with the same ConfigMap or using two ConfigMaps. +Also, you can provide in the ConfigMap a `spark-defaults.conf` file. +You can use both files without the other. + +### Production configuration + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. + +```console +$ helm install --name my-release -f ./values-production.yaml bitnami/spark +``` + +- Enable ingress controller +```diff +- ingress.enabled: false ++ ingress.enabled: true +``` + +- Enable RPC authentication and encryption: +```diff +- security.rpc.authenticationEnabled: no +- security.rpc.encryptionEnabled: no ++ security.rpc.authenticationEnabled: yes ++ security.rpc.encryptionEnabled: yes +``` + +- Enable storage encryption: +```diff +- security.storageEncryptionEnabled: no ++ security.storageEncryptionEnabled: yes +``` + +- Configure SSL parameters: +```diff +- security.ssl.enabled: no +- security.ssl.needClientAuth: no ++ security.ssl.enabled: yes ++ security.ssl.needClientAuth: yes +``` + +- Set a secret name for passwords: +```diff ++ security.passwordsSecretName: my-passwords-secret +``` + +- Set a secret name for certificates: +```diff ++ security.certificatesSecretName: my-certificates-secret +``` + +- Enable authentication between master and workers: +```diff +- auth.enabled: false ++ auth.enabled: true +``` + +- Enable SSL communication: +```diff +- ssl.enabled: false ++ ssl.enabled: true +``` + +- Enable local storage encryption: +```diff +- storageEncription.enabled: false ++ storageEncription.enabled: true +``` + +- Enable autoscaling depending on CPU: +```diff +- worker.autoscaling.enabled: false ++ worker.autoscaling.enabled: true +``` + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Submit an application + +To submit an application to the cluster use the `spark-submit` script. You can obtain the script [here](https://github.com/apache/spark/tree/master/bin). For example, to deploy one of the example applications: +```bash +$ ./bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://: --deploy-mode cluster ./examples/jars/spark-examples_2.11-2.4.3.jar 1000 +``` + +Where the master IP and port must be changed by you master IP address and port. +> Be aware that currently is not possible to submit an application to a standalone cluster if RPC authentication is configured. More info about the issue [here](https://issues.apache.org/jira/browse/SPARK-25078). + +## Enable security for spark + +### Configure ssl communication + +In order to enable secure transport between workers and master deploy the helm chart with these options: + +```console +$ helm install --name my-release --set ssl.enabled=true bitnami/spark + +``` + +### How to create the certificates secret + +It is needed to create two secrets to set the passwords and certificates. The name of the two secrets should be configured on `security.passwordsSecretName` and `security.certificatesSecretName`. To generate certificates for testing purpose you can use [this script](https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh). +Into the certificates secret, the keys must be `spark-keystore.jks` and `spark-truststore.jks`, and the content must be text on JKS format. +To generate the certificates secret, first it is needed to generate the two certificates and rename them as `spark-keystore.jks` and `spark-truststore.jks`. +Once the certificates are created, to create the secret having the file names as keys, execute a command like the following: +```bash +$ kubectl create secret generic my-secret --from-file=./keystore/spark-keystore.jks --from-file=./truststore/spark-truststore.jks +``` + +The second secret, the secret for passwords should have four keys: `rpc-authentication-secret`, `ssl-key-password`, `ssl-keystore-password` and `ssl-truststore-password`. +To create this secret, execute a command like the following: +```bash +$ kubectl create secret generic my-passwords-secret --from-literal=rpc-authentication-secret=myRPCSecret --from-literal=ssl-key-password=mySSLKeyPassword --from-literal=ssl-keystore-password=password --from-literal=ssl-truststore-password=password +``` + +Now that the two secrets are created, deploy the chart enabling security configuration and setting the name for the certificates secret (`my-secret` in this case) at the `security.certificatesSecretName` and setting the name for the passwords secret (`my-passwords-secret` in this case) at `security.passwordsSecretName`. + +To deploy execute the following: +```bash +$ helm install . --set security.certificatesSecretName=my-secret,security.passwordsSecretName=my-passwords-secret,security.rpc.authenticationEnabled=yes,security.rpc.encryptionEnabled=yes,security.storageEncrytionEnabled=yes,security.ssl.enabled=yes,security.ssl.needClientAuth=yes +``` + +> Be aware that currently is not possible to submit an application to a standalone cluster if RPC authentication is configured. More info about the issue [here](https://issues.apache.org/jira/browse/SPARK-25078). diff --git a/bitnami/spark/templates/NOTES.txt b/bitnami/spark/templates/NOTES.txt new file mode 100644 index 000000000..64a4e56b4 --- /dev/null +++ b/bitnami/spark/templates/NOTES.txt @@ -0,0 +1,42 @@ +1. Get the Spark master WebUI URL by running these commands: +{{- if .Values.ingress.enabled }} + export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ include "spark.fullname" . }}-ingress -o jsonpath='{.spec.rules[0].host}') + echo "Spark-master URL: http://$HOSTNAME/" +{{- else }} +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "spark.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "spark.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "spark.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.webPort }} +{{- else if contains "ClusterIP" .Values.service.type }} + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "spark.fullname" . }}-master-svc {{ default "8080" .Values.service.webPort }}:{{ default "8080" .Values.service.webPort }} + echo "Visit http://127.0.0.1:{{ .Values.service.webPort }} to use your application" +{{- end }} +{{- end }} + +2. Submit an application to the cluster: + + To submit an application to the cluster the spark-submit script must be used. That script can be + obtained at https://github.com/apache/spark/tree/master/bin. + + First, obtain the master IP, to do that the service type must be NodePort or LoadBalancer. Run the following command to obtain the master IP and submit your application: + +{{- if or (contains "NodePort" .Values.service.type) (contains "LoadBalancer" .Values.service.type) }} + $ export MASTER_IP=$(kubectl get services | awk '/{{ include "spark.fullname" . }}/ { print $3 }') + $ spark-submit --master spark://$MASTER_IP:{{ .Values.service.clusterPort }} --deploy-mode cluster /path/to/application 1000 +{{- else }} + $ kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "spark.fullname" . }}-master-svc {{ default "7077" .Values.service.clusterPort }}:{{ default "7077" .Values.service.clusterPort }} + $ spark-submit --master spark://127.0.0.1:{{ .Values.master.clusterPort }} --deploy-mode cluster /path/to/application 1000 +{{- end }} + +** IMPORTANT: When submit an application the --master parameter should be set to the service IP, if not, the application will not resolve the master. ** + +** Please be patient while the chart is being deployed ** + +{{ include "spark.rollingTags.warning" . }} + +{{ include "spark.validateValues" . }} diff --git a/bitnami/spark/templates/_helpers.tpl b/bitnami/spark/templates/_helpers.tpl new file mode 100644 index 000000000..4d50f742b --- /dev/null +++ b/bitnami/spark/templates/_helpers.tpl @@ -0,0 +1,100 @@ +{{- /* vim: set filetype=mustache: */}} +{{- /* +Expand the name of the chart. +*/}} +{{- define "spark.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- /* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "spark.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- /* +Create chart name and version as used by the chart label. +*/}} +{{- define "spark.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "spark.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if .Values.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if .Values.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "spark.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "spark.validateValues.workerCount" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{/* Validate values of Spark - number of workers must be greater than 0 */}} +{{- define "spark.validateValues.workerCount" -}} +{{- $replicaCount := int .Values.worker.replicaCount }} +{{- if lt $replicaCount 1 -}} +spark: workerCount + Worker replicas must be greater than 0!! + Please set a valid worker count size (--set worker.replicaCount=X) +{{- end -}} +{{- end -}} + + +{{/* Get the secret for paswords */}} +{{- define "spark.get.passwordSecretName" -}} +{{- if .Values.security.passwordsSecretName -}} + {{- printf "%s" .Values.security.passwordsSecretName -}} +{{- else }} + {{- printf "%s-secret" (include "spark.fullname" .) -}} +{{- end }} +{{- end -}} + +{{/* Warning for rolling tags */}} +{{- define "spark.rollingTags.warning" -}} +{{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .Values.image.repository }}:{{ .Values.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end -}} +{{- end -}} diff --git a/bitnami/spark/templates/headless-svc.yaml b/bitnami/spark/templates/headless-svc.yaml new file mode 100644 index 000000000..04532dd46 --- /dev/null +++ b/bitnami/spark/templates/headless-svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "spark.fullname" . }}-headless + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: ClusterIP + clusterIP: None + selector: + app.kubernetes.io/name: {{ include "spark.name" . }} + release: "{{ .Release.Name }}" diff --git a/bitnami/spark/templates/hpa-worker.yaml b/bitnami/spark/templates/hpa-worker.yaml new file mode 100644 index 000000000..d295ab098 --- /dev/null +++ b/bitnami/spark/templates/hpa-worker.yaml @@ -0,0 +1,23 @@ +{{- if .Values.worker.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/component: worker-autoscaler + name: {{ include "spark.fullname" . }}-autoscaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: statefulset + name: {{ include "spark.fullname" . }}-worker + minReplicas: {{ .Values.worker.replicaCount }} + maxReplicas: {{ .Values.worker.replicasMax }} + metrics: + - type: Resource + resource: + {{- tpl .Values.worker.autoscaling.resource | nindent 6 }} +{{- end }} diff --git a/bitnami/spark/templates/ingress.yaml b/bitnami/spark/templates/ingress.yaml new file mode 100644 index 000000000..d5775c6c0 --- /dev/null +++ b/bitnami/spark/templates/ingress.yaml @@ -0,0 +1,30 @@ +{{- if .Values.ingress.enabled }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ include "spark.fullname" . }}-ingress + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/component: ingress + annotations: + {{- if .Values.ingress.certManager }} + kubernetes.io/tls-acme: "true" + {{- end }} + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} +{{- end }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + - path: {{ default "/" .path }} + backend: + serviceName: {{ include "spark.fullname" $ }}-master-svc + servicePort: http + {{- end }} +{{- end }} diff --git a/bitnami/spark/templates/secret.yaml b/bitnami/spark/templates/secret.yaml new file mode 100644 index 000000000..5ce3250fe --- /dev/null +++ b/bitnami/spark/templates/secret.yaml @@ -0,0 +1,21 @@ +{{- if not .Values.security.passwordsSecretName }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "spark.fullname" . }}-secret + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: +{{- if eq (.Values.security.rpc.authenticationEnabled | quote ) "yes" }} + rpc-authentication-secret: {{ "RPCDefaultSecret" | b64enc | quote }} +{{- end }} +{{- if eq (.Values.security.ssl.enabled | quote ) "yes" }} + ssl-key-password: {{ randAlphaNum 10 | b64enc | quote }} + ssl-keystore-password: {{ randAlphaNum 10 | b64enc | quote }} + ssl-truststore-password: {{ randAlphaNum 10 | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/bitnami/spark/templates/statefulset-master.yaml b/bitnami/spark/templates/statefulset-master.yaml new file mode 100644 index 000000000..245816627 --- /dev/null +++ b/bitnami/spark/templates/statefulset-master.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "spark.fullname" . }}-master + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: master +spec: + serviceName: {{ template "spark.fullname" . }}-headless + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "spark.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/component: master + {{- if .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml .Values.master.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "spark.imagePullSecrets" . | nindent 6 }} + {{- if .Values.master.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.master.securityContext.fsGroup }} + runAsUser: {{ .Values.master.securityContext.runAsUser }} + {{- end }} + containers: + - name: spark-master + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.master.webPort }} + protocol: TCP + - name: cluster + containerPort: {{ .Values.master.clusterPort }} + volumeMounts: + {{- if .Values.master.configurationConfigMap }} + - name: config + mountPath: '/opt/bitnami/spark/conf/' + {{- end }} + {{- if .Values.security.certificatesSecretName }} + - name: certs + mountPath: '/opt/bitnami/spark/conf/certs' + readOnly: true + {{- end }} + env: + - name: SPARK_MODE + value: "master" + {{- if .Values.master.debug }} + - name: BASH_DEBUG + value: "1" + {{- end }} + - name: SPARK_DAEMON_MEMORY + value: {{ .Values.master.daemonMemoryLimit | quote }} + {{- if .Values.master.clusterPort }} + - name: SPARK_MASTER_PORT + value: {{ .Values.service.clusterPort | quote }} + {{- end }} + {{- if .Values.master.webPort }} + - name: SPARK_MASTER_WEBUI_PORT + value: {{ .Values.master.webPort | quote }} + {{- end }} + {{- if .Values.master.configOptions }} + - name: SPARK_MASTER_OPTS + value: {{ .Values.master.configOptions | quote }} + {{- end }} + {{- if eq ( .Values.security.rpc.authenticationEnabled | quote ) "yes" }} + - name: SPARK_RPC_AUTHENTICATION_ENABLED + value: {{ .Values.security.rpc.authenticationEnabled | quote }} + - name: SPARK_RPC_AUTHENTICATION_SECRET + valueFrom: + secretKeyRef: + {{- if .Values.security.passwordsSecretName }} + name: {{ .Values.security.passwordsSecretName }} + {{- else }} + name: {{ include "spark.fullname" . }}-secret + {{- end }} + key: rpc-authentication-secret + {{- end }} + {{- if eq ( .Values.security.encryptionEnabled | quote ) "yes" }} + - name: SPARK_RPC_ENCRYPTION_ENABLED + value: {{ .Values.security.rpc.encryptionEnabled | quote }} + {{- end }} + {{- if eq ( .Values.security.storageEncriptionEnabled | quote ) "yes" }} + - name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED + value: {{ .Values.security.storageEncryptionEnabled | quote }} + {{- end }} + {{- if eq ( .Values.security.ssl.enabled | quote ) "yes" }} + - name: SPARK_SSL_ENABLED + value: {{ .Values.security.ssl.enabled | quote }} + - name: SPARK_SSL_KEY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-key-password + - name: SPARK_SSL_KEYSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-keystore-password + - name: SPARK_SSL_TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-truststore-password + - name: SPARK_SSL_NEED_CLIENT_AUTH + value: {{ .Values.security.ssl.needClientAuth }} + - name: SPARK_SSL_PROTOCOL + value: {{ .Values.security.ssl.protocol }} + {{- end }} + {{- if .Values.master.extraEnvVars }} + {{- toYaml .Values.master.extraEnvVars | nindent 10 }} + {{- end }} + {{- if .Values.master.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.master.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.master.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.master.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.master.resources | nindent 12 }} + volumes: + {{- if .Values.master.configurationConfigMap }} + - name: config + configMap: + name: {{ .Values.master.configurationConfigMap }} + {{- end }} + {{- if .Values.security.certificatesSecretName }} + - name: certs + secret: + secretName: {{ .Values.security.certificatesSecretName }} + {{- end }} diff --git a/bitnami/spark/templates/statefulset-worker.yaml b/bitnami/spark/templates/statefulset-worker.yaml new file mode 100644 index 000000000..8d556fc9f --- /dev/null +++ b/bitnami/spark/templates/statefulset-worker.yaml @@ -0,0 +1,188 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "spark.fullname" . }}-worker + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: worker +spec: + serviceName: {{ template "spark.fullname" . }}-headless + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "spark.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: worker + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/component: worker + {{- if .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml .Values.worker.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "spark.imagePullSecrets" . | nindent 6 }} + {{- if .Values.worker.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.worker.securityContext.fsGroup }} + runAsUser: {{ .Values.worker.securityContext.runAsUser }} + {{- end }} + containers: + - name: spark-worker + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.worker.webPort }} + protocol: TCP + {{- if .Values.worker.clusterPort }} + - name: cluster + containerPort: {{ .Values.worker.clusterPort }} + {{- end }} + volumeMounts: + {{- if .Values.worker.configurationConfigMap }} + - name: config + mountPath: '/opt/bitnami/spark/conf/' + {{- end }} + {{- if .Values.security.certificatesSecret }} + - name: certs + mountPath: '/opt/bitnami/spark/conf/certs' + readOnly: true + {{- end }} + env: + - name: SPARK_MODE + value: "worker" + {{- if .Values.worker.debug }} + - name: BASH_DEBUG + value: "1" + {{- end }} + - name: SPARK_DAEMON_MEMORY + value: {{ .Values.worker.daemonMemoryLimit | quote }} + ## There are some environment variables whose existence needs + ## to be checked because Spark checks if they are null instead of an + ## empty string + {{- if .Values.worker.memoryLimit }} + - name: SPARK_WORKER_MEMORY + value: {{ .Values.worker.memoryLimit | quote }} + {{- end }} + {{- if .Values.worker.coreLimit }} + - name: SPARK_WORKER_CORES + value: {{ .Values.worker.coreLimit | quote }} + {{- end }} + {{- if .Values.worker.clusterPort }} + - name: SPARK_WORKER_PORT + value: {{ .Values.worker.clusterPort | quote }} + {{- end }} + {{- if .Values.worker.webPort }} + - name: SPARK_WORKER_WEBUI_PORT + value: {{ .Values.worker.webPort | quote }} + {{- end }} + {{- if .Values.worker.dir }} + - name: SPARK_WORKER_DIR + value: {{ .Values.worker.dir | quote }} + {{- end }} + - name: SPARK_DAEMON_JAVA_OPTS + value: {{ .Values.worker.javaOptions | quote }} + - name: SPARK_MASTER_URL + value: spark://{{ include "spark.fullname" . }}-master-svc:{{ .Values.service.clusterPort }} + # If you use a custom properties file, it must be loaded using a ConfigMap + - name: SPARK_WORKER_OPTS + value: {{ .Values.worker.configOptions }} + {{- if eq ( .Values.security.rpc.authenticationEnabled | quote ) "yes" }} + - name: SPARK_RPC_AUTHENTICATION_ENABLED + value: {{ .Values.security.rpc.authenticationEnabled | quote }} + - name: SPARK_RPC_AUTHENTICATION_SECRET + valueFrom: + secretKeyRef: + {{- if .Values.security.passwordsSecretName }} + name: {{ .Values.security.passwordsSecretName }} + key: rpc-authentication-secret + {{- else }} + name: {{ include "spark.fullname" . }}-secret + key: rpc-authentication-secret + {{- end }} + {{- end }} + {{- if eq ( .Values.security.encryptionEnabled | quote ) "yes" }} + - name: SPARK_RPC_ENCRYPTION_ENABLED + value: {{ .Values.security.rpc.encryptionEnabled | quote }} + {{- end }} + {{- if eq ( .Values.security.storageEncriptionEnabled | quote ) "yes" }} + - name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED + value: {{ .Values.security.storageEncryptionEnabled | quote }} + {{- end }} + {{- if eq ( .Values.security.ssl.enabled | quote ) "yes" }} + - name: SPARK_SSL_ENABLED + value: {{ .Values.security.ssl.enabled | quote }} + - name: SPARK_SSL_KEY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-key-password + - name: SPARK_SSL_KEYSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-keystore-password + - name: SPARK_SSL_TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "spark.get.passwordSecretName" }} + key: ssl-truststore-password + - name: SPARK_SSL_NEED_CLIENT_AUTH + value: {{ .Values.security.ssl.needClientAuth }} + - name: SPARK_SSL_PROTOCOL + value: {{ .Values.security.ssl.protocol }} + {{- end }} + {{- if .Values.master.extraEnvVars }} + {{- toYaml .Values.master.extraEnvVars | nindent 10 }} + {{- end }} + {{- if .Values.worker.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: {{ .Values.worker.webPort }} + initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: / + port: {{ .Values.worker.webPort }} + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + volumes: + {{- if .Values.worker.configurationConfigMap }} + - name: config + configMap: + name: {{ .Values.worker.configurationConfigMap }} + {{- end }} + {{- if .Values.security.certificatesSecretName }} + - name: certs + secret: + secretName: {{ .Values.security.certificatesSecretName }} + {{- end }} diff --git a/bitnami/spark/templates/svc-master.yaml b/bitnami/spark/templates/svc-master.yaml new file mode 100644 index 000000000..55a908459 --- /dev/null +++ b/bitnami/spark/templates/svc-master.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "spark.fullname" . }}-master-svc + labels: + app.kubernetes.io/name: {{ include "spark.name" . }} + helm.sh/chart: {{ include "spark.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + {{- range $key, $value := .Values.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if eq .Values.service.type "LoadBalancer" }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- end }} + ports: + - port: {{ .Values.service.clusterPort }} + targetPort: cluster + name: cluster + - port: {{ .Values.service.webPort }} + targetPort: http + name: http + protocol: TCP + selector: + app.kubernetes.io/name: {{ include "spark.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master diff --git a/bitnami/spark/values-production.yaml b/bitnami/spark/values-production.yaml new file mode 100644 index 000000000..a610228a0 --- /dev/null +++ b/bitnami/spark/values-production.yaml @@ -0,0 +1,259 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName + +## Bitnami Spark image version +## ref: https://hub.docker.com/r/bitnami/spark/tags/ +## +image: + registry: docker.io + repository: bitnami/spark + tag: 2.4.3-debian-9-r13 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + + ## Pull secret for this image + # pullSecrets: + # - myRegistryKeySecretName + + +## Spark Components configuration +## +master: + ## Spark master specific configuration + + ## Set a custom configuration by using an existing configMap with the configuration file. + # configurationConfigMap: + + webPort: 8080 + clusterPort: 7077 + + ## Set the master daemon memory limit. + # daemonMemoryLimit: + + ## Use a string to set the config options for in the form "-Dx=y" + # configOptions: + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: SPARK_DAEMON_JAVA_OPTS + ## value: -Dx=y + # extraEnvVars: + + ## Kubernetes Security Context + ## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## Tolerations for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Configure extra options for liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +worker: + ## Spark worker specific configuration + + ## Set a custom configuration by using an existing configMap with the configuration file. + # configurationConfigMap: + + webPort: 8081 + ## Set to true to use a custom cluster port instead of a random port. + # clusterPort: + + ## Set the daemonMemoryLimit as the daemon max memory + # daemonMemoryLimit: + + ## Set the worker memory limit + # memoryLimit: + + ## Set the maximun number of cores + # coreLimit: + + ## Working directory for the application + # dir: + + ## Options for the JVM as "-Dx=y" + # javaOptions: + + ## Configuraion options in the form "-Dx=y" + # configOptions: + + ## Number of spark workers (will be the min number when autoscaling is enabled) + replicaCount: 2 + + autoscaling: + ## Enable replica autoscaling depending on CPU + enabled: true + resource: + + ## Max number of workers when using autoscaling + # replicasMax: + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: SPARK_DAEMON_JAVA_OPTS + ## value: -Dx=y + # extraEnvVars: + + ## Kubernetes Security Context + ## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## Tolerations for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Configure extra options for liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## Security configuration +security: + ## Name of the secret that contains all the passwords. This is optional, by default random passwords are generated. + # passwordsSecretName: + + ## RPC configuration + rpc: + authenticationEnabled: yes + encryptionEnabled: yes + + ## Enables local storage encryption + storageEncryptionEnabled: yes + + ## SSL configuration + ssl: + enabled: yes + needClientAuth: yes + protocol: TLSv1.2 + + ## Name of the secret that contains the certificates + ## It should contains two keys called "spark-keystore.jks" and "spark-truststore.jks" with the files in JKS format. + # certificatesSecretName: + + +## Service to access the master from the workers and to the WebUI +## +service: + type: ClusterIP + clusterPort: 7077 + webPort: 8080 + ## Specify the NodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Use loadBalancerIP to request a specific static IP, + # loadBalancerIP: + + ## Service annotations done as key:value pairs + annotations: + +## Ingress controller to access the web UI. +ingress: + enabled: true + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + annotations: + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: spark.local + path: / diff --git a/bitnami/spark/values.yaml b/bitnami/spark/values.yaml new file mode 100644 index 000000000..b33cd9a4a --- /dev/null +++ b/bitnami/spark/values.yaml @@ -0,0 +1,259 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName + +## Bitnami Spark image version +## ref: https://hub.docker.com/r/bitnami/spark/tags/ +## +image: + registry: docker.io + repository: bitnami/spark + tag: 2.4.3-debian-9-r13 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + + ## Pull secret for this image + # pullSecrets: + # - myRegistryKeySecretName + + +## Spark Components configuration +## +master: + ## Spark master specific configuration + + ## Set a custom configuration by using an existing configMap with the configuration file. + # configurationConfigMap: + + webPort: 8080 + clusterPort: 7077 + + ## Set the master daemon memory limit. + # daemonMemoryLimit: + + ## Use a string to set the config options for in the form "-Dx=y" + # configOptions: + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: SPARK_DAEMON_JAVA_OPTS + ## value: -Dx=y + # extraEnvVars: + + ## Kubernetes Security Context + ## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## Tolerations for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Configure extra options for liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +worker: + ## Spark worker specific configuration + + ## Set a custom configuration by using an existing configMap with the configuration file. + # configurationConfigMap: + + webPort: 8081 + ## Set to true to use a custom cluster port instead of a random port. + # clusterPort: + + ## Set the daemonMemoryLimit as the daemon max memory + # daemonMemoryLimit: + + ## Set the worker memory limit + # memoryLimit: + + ## Set the maximun number of cores + # coreLimit: + + ## Working directory for the application + # dir: + + ## Options for the JVM as "-Dx=y" + # javaOptions: + + ## Configuraion options in the form "-Dx=y" + # configOptions: + + ## Number of spark workers (will be the min number when autoscaling is enabled) + replicaCount: 2 + + autoscaling: + ## Enable replica autoscaling depending on CPU + enabled: false + resource: + + ## Max number of workers when using autoscaling + # replicasMax: + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: SPARK_DAEMON_JAVA_OPTS + ## value: -Dx=y + # extraEnvVars: + + ## Kubernetes Security Context + ## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + ## Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + ## Tolerations for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + ## Affinity for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Configure extra options for liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## Security configuration +security: + ## Name of the secret that contains all the passwords. This is optional, by default random passwords are generated. + # passwordsSecretName: + + ## RPC configuration + rpc: + authenticationEnabled: no + encryptionEnabled: no + + ## Enables local storage encryption + storageEncryptionEnabled: no + + ## SSL configuration + ssl: + enabled: no + needClientAuth: no + protocol: TLSv1.2 + + ## Name of the secret that contains the certificates + ## It should contains two keys called "spark-keystore.jks" and "spark-truststore.jks" with the files in JKS format. + # certificatesSecretName: + + +## Service to access the master from the workers and to the WebUI +## +service: + type: ClusterIP + clusterPort: 7077 + webPort: 8080 + ## Specify the NodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Use loadBalancerIP to request a specific static IP, + # loadBalancerIP: + + ## Service annotations done as key:value pairs + annotations: + +## Ingress controller to access the web UI. +ingress: + enabled: false + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + annotations: + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: spark.local + path: /