From d7a0ac8fb0171c1cd18faeeb3e16d672f35bc169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibone=20Gonz=C3=A1lez=20Mauraza?= Date: Thu, 20 Feb 2020 11:59:30 +0100 Subject: [PATCH] [bitnami/airflow] separate resources (#1951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add separate resources for worker, web and scheduler for airflow * update chart version * change worker replicas to 1 * change version, in readme set the default values, values.yaml and values-production.yaml * Add if in resources and separate limits and requests * Fix typpo in readme * Delete the old resources, delete a blank line in values-production and in values look in the same way to values-production * Changes in values-production and values Co-authored-by: Carlos Rodríguez Hernández --- bitnami/airflow/README.md | 9 ++- .../templates/deployment-scheduler.yaml | 4 +- bitnami/airflow/templates/deployment-web.yaml | 4 +- .../airflow/templates/metrics-deployment.yaml | 2 + .../airflow/templates/statefulset-worker.yaml | 4 +- bitnami/airflow/values-production.yaml | 61 +++++++++++++++---- bitnami/airflow/values.yaml | 61 +++++++++++++++---- 7 files changed, 118 insertions(+), 27 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 125ad8364..d7654172e 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -93,12 +93,18 @@ The following tables lists the configurable parameters of the Airflow chart and | `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | +| `airflow.worker.resources.limits` | The resources limits for the Worker containers | `{}` | +| `airflow.worker.resources.requests` | The requested resources for the Worker containers | `{}` | | `airflow.auth.forcePassword` | Force users to specify a password | `false` | | `airflow.auth.username` | Username to access web UI | `user` | | `airflow.auth.password` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | +| `airflow.web.resources.limits` | The resources limits for the web containers | `{}` | +| `airflow.web.resources.requests` | The requested resources for the web containers | `{}` | +| `airflow.scheduler.resources.limits` | The resources limits for the scheduler containers | `{}` | +| `airflow.scheduler.resources.requests` | The requested resources for the scheduler containers | `{}` | | `airflow.webserverConfigMap` | Config map name for ~/airflow/webserver_config.py | `nil` | | `securityContext.enabled` | Enable security context | `true` | | `securityContext.fsGroup` | Group ID for the container | `1001` | @@ -106,7 +112,7 @@ The following tables lists the configurable parameters of the Airflow chart and | `service.type` | Kubernetes Service type | `ClusterIP` | | `service.port` | Airflow Web port | `8080` | | `service.nodePort` | Kubernetes Service nodePort | `nil` | -| `service.loadBalancerIP` | loadBalancerIP for Airflow Service | `nil` | +| `service.loadBalancerIP` | loadBalancerIP for Airflow Service | `nil` | | `service.annotations` | Service annotations | `` | | `ingress.enabled` | Enable ingress controller resource | `false` | | `ingress.certManager` | Add annotations for cert-manager | `false` | @@ -122,7 +128,6 @@ The following tables lists the configurable parameters of the Airflow chart and | `nodeSelector` | Node labels for pod assignment | `{}` | | `tolerations` | Toleration labels for pod assignment | `[]` | | `affinity` | Map of node/pod affinities | `{}` | -| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | | `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | | `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 180 | | `livenessProbe.periodSeconds` | How often to perform the probe | 20 | diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index a4e0de070..47efa4f21 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -211,7 +211,9 @@ spec: mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg {{- end }} - resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.airflow.scheduler.resources }} + resources: {{- toYaml .Values.airflow.scheduler.resources | nindent 12 }} + {{- end }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index 78ab215d0..5f7f708c5 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -246,7 +246,9 @@ spec: mountPath: /opt/bitnami/airflow/webserver_config.py subPath: webserver_config.py {{- end }} - resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.airflow.web.resources }} + resources: {{- toYaml .Values.airflow.web.resources | nindent 12 }} + {{- end }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/metrics-deployment.yaml b/bitnami/airflow/templates/metrics-deployment.yaml index 165357262..39fb65795 100644 --- a/bitnami/airflow/templates/metrics-deployment.yaml +++ b/bitnami/airflow/templates/metrics-deployment.yaml @@ -62,5 +62,7 @@ spec: ports: - name: metrics containerPort: 9112 + {{- if .Values.metrics.resources}} resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} {{- end }} diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index 547d7be2a..59019a53b 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -102,7 +102,9 @@ spec: - name: airflow-worker image: "{{ template "airflow.workerImage" . }}" imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}" - resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.airflow.worker.resources }} + resources: {{- toYaml .Values.airflow.worker.resources | nindent 12 }} + {{- end }} env: {{- if .Values.workerImage.debug }} - name: BASH_DEBUG diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index ae78393da..69c4ccdd2 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -148,6 +148,56 @@ airflow: worker: port: 8793 replicas: 3 + ## Worker containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m + + ## Airflow web specific configuration + # + web: + ## Web container' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m + + ## Airflow scheduler specific configuration + # + scheduler: + ## Scheduler containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html @@ -276,17 +326,6 @@ tolerations: [] ## affinity: {} -## Configure resource requests and limits -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ -## -resources: -# limits: -# cpu: 200m -# memory: 1Gi -# requests: -# memory: 256Mi -# cpu: 250m - ## Configure extra options for liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) livenessProbe: diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 1b5158672..edce388ec 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -148,6 +148,56 @@ airflow: worker: port: 8793 replicas: 1 + ## Worker containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m + + ## Airflow web specific configuration + # + web: + ## Web container' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m + + ## Airflow scheduler specific configuration + # + scheduler: + ## Scheduler containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html @@ -276,17 +326,6 @@ tolerations: [] ## affinity: {} -## Configure resource requests and limits -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ -## -resources: -# limits: -# cpu: 200m -# memory: 1Gi -# requests: -# memory: 256Mi -# cpu: 250m - ## Configure extra options for liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) livenessProbe: