From 55e58ce45682451dfdc9b3f62bd142af46d937f4 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Tue, 30 Apr 2019 18:02:17 +0200 Subject: [PATCH 01/11] Add airflow chart --- bitnami/airflow/.helmignore | 1 + bitnami/airflow/Chart.yaml | 18 + bitnami/airflow/README.md | 183 ++++++++++ bitnami/airflow/files/dags/REAMDE.md | 1 + bitnami/airflow/requirements.lock | 9 + bitnami/airflow/requirements.yaml | 9 + bitnami/airflow/templates/NOTES.txt | 22 ++ bitnami/airflow/templates/_helpers.tpl | 189 +++++++++++ bitnami/airflow/templates/configmap-dags.yaml | 13 + bitnami/airflow/templates/configmap.yaml | 14 + .../templates/deployment-scheduler.yaml | 177 ++++++++++ bitnami/airflow/templates/deployment-web.yaml | 207 ++++++++++++ bitnami/airflow/templates/ingress.yaml | 33 ++ .../airflow/templates/secret-externaldb.yaml | 14 + bitnami/airflow/templates/secret-redis.yaml | 14 + bitnami/airflow/templates/secrets.yaml | 23 ++ .../airflow/templates/statefulset-worker.yaml | 192 +++++++++++ .../templates/svc-headless-worker.yaml | 20 ++ bitnami/airflow/templates/svc.yaml | 31 ++ bitnami/airflow/values.yaml | 315 ++++++++++++++++++ 20 files changed, 1485 insertions(+) create mode 100644 bitnami/airflow/.helmignore create mode 100644 bitnami/airflow/Chart.yaml create mode 100644 bitnami/airflow/README.md create mode 100644 bitnami/airflow/files/dags/REAMDE.md create mode 100644 bitnami/airflow/requirements.lock create mode 100644 bitnami/airflow/requirements.yaml create mode 100644 bitnami/airflow/templates/NOTES.txt create mode 100644 bitnami/airflow/templates/_helpers.tpl create mode 100644 bitnami/airflow/templates/configmap-dags.yaml create mode 100644 bitnami/airflow/templates/configmap.yaml create mode 100644 bitnami/airflow/templates/deployment-scheduler.yaml create mode 100644 bitnami/airflow/templates/deployment-web.yaml create mode 100644 bitnami/airflow/templates/ingress.yaml create mode 100644 bitnami/airflow/templates/secret-externaldb.yaml create mode 100644 bitnami/airflow/templates/secret-redis.yaml create mode 100644 bitnami/airflow/templates/secrets.yaml create mode 100644 bitnami/airflow/templates/statefulset-worker.yaml create mode 100644 bitnami/airflow/templates/svc-headless-worker.yaml create mode 100644 bitnami/airflow/templates/svc.yaml create mode 100644 bitnami/airflow/values.yaml diff --git a/bitnami/airflow/.helmignore b/bitnami/airflow/.helmignore new file mode 100644 index 000000000..6b8710a71 --- /dev/null +++ b/bitnami/airflow/.helmignore @@ -0,0 +1 @@ +.git diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml new file mode 100644 index 000000000..3a5cd1021 --- /dev/null +++ b/bitnami/airflow/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +name: airflow +version: 0.0.1 +appVersion: 1.10.3 +description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows. +keywords: +- apache +- airflow +- workflow +- dag +home: https://airflow.apache.org/ +sources: +- https://github.com/bitnami/bitnami-docker-airflow +maintainers: +- name: Bitnami + email: containers@bitnami.com +engine: gotpl +icon: https://bitnami.com/assets/stacks/airflow/img/airflow-stack-110x117.png diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md new file mode 100644 index 000000000..112415516 --- /dev/null +++ b/bitnami/airflow/README.md @@ -0,0 +1,183 @@ +# Apache Airflow + +[Apache Airflow]() is a platform to programmatically author, schedule and monitor workflows. + +## TL;DR; + +```console +$ helm install bitnami/airflow +``` + +## Introduction + +This chart bootstraps an [Apache Airflow](https://github.com/bitnami/bitnami-docker-airflow) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.4+ with Beta APIs enabled +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install --name my-release bitnami/airflow +``` + +The command deploys Airflow on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the Kafka chart and their default values. + +| Parameter | Description | Default | +| ----------------------------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------- | +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.registry` | Airflow image registry | `docker.io` | +| `image.repository` | Airflow image name | `bitnami/airflow` | +| `image.tag` | Airflow image tag | `{VERSION}` | +| `image.pullPolicy` | Airflow image pull policy | `Always` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `schedulerImage.registry` | Airflow Scheduler image registry | `docker.io` | +| `schedulerImage.repository` | Airflow Scheduler image name | `bitnami/airflow-shceduler` | +| `schedulerImage.tag` | Airflow Scheduler image tag | `{VERSION}` | +| `schedulerImage.pullPolicy` | Airflow Scheduler image pull policy | `Always` | +| `schedulerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `schedulerImage.debug` | Specify if debug values should be set | `false` | +| `workerImage.registry` | Airflow Worker image registry | `docker.io` | +| `workerImage.repository` | Airflow Worker image name | `bitnami/airflow-worker` | +| `workerImage.tag` | Airflow Worker image tag | `{VERSION}` | +| `workerImage.pullPolicy` | Airflow Worker image pull policy | `Always` | +| `workerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `workerImage.debug` | Specify if debug values should be set | `false` | +| `gitImage.registry` | Git image registry | `docker.io` | +| `gitImage.repository` | Git image name | `bitnami/git` | +| `gitImage.tag` | Git image tag | `{VERSION}` | +| `gitImage.pullPolicy` | Git image pull policy | `Always` | +| `gitImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `airflow.airflowConfigurationConfigMap` | Name of an existing config map containing the Airflow config file | `nil` | +| `airflow.dagsConfigMap` | Name of an existing config map containing all the DAGs files you want to load in Airflow. | `nil` | +| `airflow.loadExamples` | Switch to load some Airflow examples | `true` | +| `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | +| `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from | `nil` | +| `airflow.cloneDagFilesFromGit.revision` | Revision from repository to checkout | `nil` | +| `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | +| `airflow.worker.port` | Airflow Worker port | `8793` | +| `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | +| `airflow.auth.airflowUsername` | Username to access web UI | `user` | +| `airflow.auth.airflowPassword` | Password to access web UI | `nil` | +| `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | +| `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | +| `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the container | `1001` | +| `securityContext.runAsUser` | User ID for the container | `1001` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Airflow Web port | `8080` | +| `service.nodePort` | Kubernetes Service nodePort | `nil` | +| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | +| `service.annotations` | Service annotations | `` | +| `ingress.enabled` | Enable ingress controller resource | `false` | +| `ingress.certManager` | Add annotations for cert-manager | `false` | +| `ingress.annotations` | Ingress annotations | `[]` | +| `ingress.hosts[0].name` | Hostname to your Wordpress installation | `airflow.local` | +| `ingress.hosts[0].path` | Path within the url structure | `/` | +| `ingress.tls[0].hosts[0]` | TLS hosts | `airflow.local` | +| `ingress.tls[0].secretName` | TLS Secret (certificates) | `airflow.local-tls` | +| `ingress.secrets[0].name` | TLS Secret Name | `nil` | +| `ingress.secrets[0].certificate` | TLS Secret Certificate | `nil` | +| `ingress.secrets[0].key` | TLS Secret Key | `nil` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | +| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 180 | +| `livenessProbe.periodSeconds` | How often to perform the probe | 20 | +| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | +| `readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | +| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | +| `postgresql.postgresqlUsername` | Airflow Postgresql username | `bn_airflow` | +| `postgresql.postgresqlPassword` | Airflow Postgresql password | `nil` | +| `postgresql.postgresqlDatabase` | Airflow Postgresql database | `bitnami_airflow` | +| `externalDatabase.host` | External PostgreSQL host | `nil` | +| `externalDatabase.user` | External PostgreSQL user | `nil` | +| `externalDatabase.password` | External PostgreSQL password | `nil` | +| `externalDatabase.database` | External PostgreSQL database name | `nil` | +| `externalDatabase.port` | External PostgreSQL port | `nil` | +| `redis.enabled` | Switch to enable or disable the Redis helm chart | `true` | +| `externalRedis.host` | External Redis host | `nil` | +| `externalRedis.port` | External Redis port | `nil` | +| `externalRedis.password` | External Redis password | `nil` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name my-release \ + --set airflow.auth.airflowUsername=my-user,airflow.auth.airflowPassword=my-passsword \ + bitnami/airflow +``` + +The above command sets the credentials to access the Airflow web UI. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install --name my-release -f values.yaml bitnami/airflow +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Persistence + +The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means that Airflow does not persist anything. + +## Load DAG files + +There are three different ways to load your custom DAG files into the Airflow chart: + +### Option 1: Load locally from the `files` folder + +If you plan to deploy the chart from your filesystem, you can copy your DAG files inside the `files/dags` directory. A config map will be created with those files and it will be mounted in all airflow nodes.. + +### Option 2: Specify an existing config map + +You can manually create a config map containing all your DAG files and then pass the name when deploying Airflow chart. For that, you can pass the option `--set airflow.dagsConfigMap`. + +### Option 3: Get your DAG files from a git repository + +You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. In order to do that, you can deploy airflow with the following options: + +```console +helm install --name my-release bitnami/airflow \ + --set airflow.cloneDagFilesFromGit.enabled=true \ + --set airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY \ + --set airflow.cloneDagFilesFromGit.revision=master + +``` \ No newline at end of file diff --git a/bitnami/airflow/files/dags/REAMDE.md b/bitnami/airflow/files/dags/REAMDE.md new file mode 100644 index 000000000..efb807bdc --- /dev/null +++ b/bitnami/airflow/files/dags/REAMDE.md @@ -0,0 +1 @@ +You can copy here your DAGs files so they are mounted at "/opt/bitnami/airflow/dags" inside the docker image. diff --git a/bitnami/airflow/requirements.lock b/bitnami/airflow/requirements.lock new file mode 100644 index 000000000..69a8a58b1 --- /dev/null +++ b/bitnami/airflow/requirements.lock @@ -0,0 +1,9 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 3.18.2 +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 6.4.5 +digest: sha256:f8531d43c9fce57c6c4c396d76bc752cc0065783bb18cba3b6bae50558e1a473 +generated: 2019-04-23T11:12:57.062256486+02:00 diff --git a/bitnami/airflow/requirements.yaml b/bitnami/airflow/requirements.yaml new file mode 100644 index 000000000..0a819c03a --- /dev/null +++ b/bitnami/airflow/requirements.yaml @@ -0,0 +1,9 @@ +dependencies: +- name: postgresql + version: 3.x.x + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled +- name: redis + version: 6.x.x + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled diff --git a/bitnami/airflow/templates/NOTES.txt b/bitnami/airflow/templates/NOTES.txt new file mode 100644 index 000000000..5c0323989 --- /dev/null +++ b/bitnami/airflow/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the Airflow URL by running: + +{{- if eq .Values.service.type "ClusterIP" }} + + echo URL : http://127.0.0.1:{{ default "8080" .Values.service.port }} + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "airflow.fullname" . }} {{ default "8080" .Values.service.port }}:8080 + +{{- else if eq .Values.service.type "NodePort" }} + export APP_BASE_URL=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export APP_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }}) + + echo URL : $APP_BASE_URL + +{{- else }} + + echo URL : http://{{ .Values.airflow.baseUrl }} +{{- end }} + +2. Get your Airflow login credentials by running: + + echo User: {{ .Values.airflow.auth.airflowUsername }} + echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) \ No newline at end of file diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl new file mode 100644 index 000000000..b30984243 --- /dev/null +++ b/bitnami/airflow/templates/_helpers.tpl @@ -0,0 +1,189 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "airflow.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 24 -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airflow.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the proper Airflow image name +*/}} +{{- define "airflow.image" -}} +{{- $registryName := .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Airflow Scheduler image name +*/}} +{{- define "airflow.schedulerImage" -}} +{{- $registryName := .Values.schedulerImage.registry -}} +{{- $repositoryName := .Values.schedulerImage.repository -}} +{{- $tag := .Values.schedulerImage.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Airflow Worker image name +*/}} +{{- define "airflow.workerImage" -}} +{{- $registryName := .Values.workerImage.registry -}} +{{- $repositoryName := .Values.workerImage.repository -}} +{{- $tag := .Values.workerImage.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper git image name +*/}} +{{- define "git.image" -}} +{{- $registryName := .Values.git.registry -}} +{{- $repositoryName := .Values.git.repository -}} +{{- $tag := .Values.git.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "airflow.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if or .Values.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if or .Values.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +Create a default fully qualified postgresql name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airflow.postgresql.fullname" -}} +{{- $name := default "postgresql" .Values.postgresql.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- end -}} + +Create a default fully qualified redis name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airflow.redis.fullname" -}} +{{- $name := default "redis" .Values.redis.nameOverride -}} +{{- printf "%s-%s-master" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- end -}} + +Create a template for the redis secret +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "airflow.redis.secretName" -}} +{{- $name := default "redis" .Values.redis.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- end -}} + +{{ template "airflow.dagFilesConfigMap" . }} +{{/* +Get the DAG files ConfigMap name. +*/}} +{{- define "airflow.dagFilesConfigMap" -}} +{{- if .Values.airflow.dagsConfigMap -}} +{{- printf "%s" .Values.airflow.dagsConfigMap -}} +{{- else -}} +{{- printf "%s-dag-files" (include "airflow.fullname" .) -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/bitnami/airflow/templates/configmap-dags.yaml b/bitnami/airflow/templates/configmap-dags.yaml new file mode 100644 index 000000000..3dab10817 --- /dev/null +++ b/bitnami/airflow/templates/configmap-dags.yaml @@ -0,0 +1,13 @@ +{{- if and (.Files.Glob "files/dags/*.py") (not .Values.airflow.dagsConfigMap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow.fullname" . }}-dag-files + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: +{{ (.Files.Glob "files/dags/*.{py}").AsConfig | indent 2 }} +{{ end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/configmap.yaml b/bitnami/airflow/templates/configmap.yaml new file mode 100644 index 000000000..f175aeb9e --- /dev/null +++ b/bitnami/airflow/templates/configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.config }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow.fullname" . }}-configuration + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + airflow.cfg: |- +{{ .Values.config | indent 4 }} +{{- end -}} \ No newline at end of file diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml new file mode 100644 index 000000000..bc5a701ee --- /dev/null +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -0,0 +1,177 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "airflow.fullname" . }}-scheduler + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + component: scheduler +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: scheduler + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: scheduler + {{- if .Values.podAnnotations }} + annotations: + {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: + {{ toYaml .Values.nodeSelector | nindent 8 }} + {{- end -}} + {{- with .Values.affinity }} + affinity: + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "airflow.imagePullSecrets" . | nindent 6 }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + initContainers: + - name: git-clone-repository + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + volumeMounts: + - name: custom-dag-files + mountPath: /dags + {{- end }} + containers: + - name: airflow-scheduler + image: {{ template "airflow.schedulerImage" . }} + imagePullPolicy: {{ .Values.schedulerImage.pullPolicy | quote }} + env: + {{- if .Values.schedulerImage.debug }} + - name: BASH_DEBUG + value: "1" + - name: NAMI_DEBUG + value: "1" + - name: NAMI_LOG_LEVEL + value: "trace8" + {{- end }} + - name: AIRFLOW_DATABASE_NAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.user | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "airflow.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: AIRFLOW_DATABASE_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "airflow.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PORT_NUMBER + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port | quote }} + {{- end }} + - name: REDIS_HOST + {{- if .Values.redis.enabled }} + value: {{ template "airflow.redis.fullname" . }} + {{- else }} + value: {{ .Values.externalRedis.host | quote }} + {{- end }} + - name: REDIS_PORT_NUMBER + {{- if .Values.redis.enabled }} + value: "6379" + {{- else }} + value: {{ .Values.externalRedis.port | quote }} + {{- end }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.redis.enabled }} + name: {{ template "airflow.redis.secretName" . }} + key: redis-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externalredis" }} + key: redis-password + {{- end }} + - name: AIRFLOW_EXECUTOR + value: "CeleryExecutor" + - name: AIRFLOW_FERNET_KEY + valueFrom: + secretKeyRef: + name: {{ if .Values.airflow.auth.existingSecret }}{{ .Values.airflow.auth.existingSecret }}{{ else }}{{ template "airflow.fullname" . }}{{ end }} + key: airflow-fernetKey + - name: AIRFLOW_WEBSERVER_HOST + value: {{ template "airflow.fullname" . }} + - name: AIRFLOW_LOAD_EXAMPLES + {{- if .Values.airflow.loadExamples }} + value: "yes" + {{- else }} + value: "no" + {{- end }} + {{- if .Values.airflow.extraEnvVars }} + {{ toYaml .Values.airflow.extraEnvVars | nindent 8 }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + {{- end }} + volumeMounts: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + mountPath: /opt/bitnami/airflow/dags + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + mountPath: /opt/bitnami/airflow/airflow.cfg + subPath: airflow.cfg + {{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + volumes: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + configMap: + name: {{ template "airflow.dagFilesConfigMap" . }} + {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + configMap: + name: {{ .Values.airflow.airflowConfigurationConfigMap }} + {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml new file mode 100644 index 000000000..5d1a8304b --- /dev/null +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -0,0 +1,207 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "airflow.fullname" . }}-web + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + component: web +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: web + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: web + {{- if .Values.podAnnotations }} + annotations: + {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: + {{ toYaml .Values.nodeSelector | nindent 8 }} + {{- end -}} + {{- with .Values.affinity }} + affinity: + {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + {{- include "airflow.imagePullSecrets" . | nindent 6 }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + initContainers: + - name: git-clone-repository + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + volumeMounts: + - name: custom-dag-files + mountPath: /dags + {{- end }} + containers: + - name: airflow-web + image: {{ template "airflow.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + env: + {{- if .Values.image.debug }} + - name: BASH_DEBUG + value: "1" + - name: NAMI_DEBUG + value: "1" + - name: NAMI_LOG_LEVEL + value: "trace8" + {{- end }} + - name: AIRFLOW_DATABASE_NAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.user | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "airflow.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: AIRFLOW_DATABASE_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "airflow.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PORT_NUMBER + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port | quote }} + {{- end }} + - name: REDIS_HOST + {{- if .Values.redis.enabled }} + value: {{ template "airflow.redis.fullname" . }} + {{- else }} + value: {{ .Values.externalRedis.host | quote }} + {{- end }} + - name: REDIS_PORT_NUMBER + {{- if .Values.redis.enabled }} + value: "6379" + {{- else }} + value: {{ .Values.externalRedis.port | quote }} + {{- end }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.redis.enabled }} + name: {{ template "airflow.redis.secretName" . }} + key: redis-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externalredis" }} + key: redis-password + {{- end }} + - name: AIRFLOW_EXECUTOR + value: "CeleryExecutor" + - name: AIRFLOW_USERNAME + value: {{ .Values.airflow.auth.airflowUsername }} + - name: AIRFLOW_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.airflow.auth.existingSecret }}{{ .Values.airflow.auth.existingSecret }}{{ else }}{{ template "airflow.fullname" . }}{{ end }} + key: airflow-password + - name: AIRFLOW_FERNET_KEY + valueFrom: + secretKeyRef: + name: {{ if .Values.airflow.auth.existingSecret }}{{ .Values.airflow.auth.existingSecret }}{{ else }}{{ template "airflow.fullname" . }}{{ end }} + key: airflow-fernetKey + - name: AIRFLOW_WEBSERVER_HOST + value: {{ template "airflow.fullname" . }} + {{- if .Values.airflow.baseUrl }} + - name: AIRFLOW_BASE_URl + value: {{ .Values.airflow.baseUrl }} + {{- end }} + - name: AIRFLOW_LOAD_EXAMPLES + {{- if .Values.airflow.loadExamples }} + value: "yes" + {{- else }} + value: "no" + {{- end }} + {{- if .Values.airflow.extraEnvVars }} + {{ toYaml .Values.airflow.extraEnvVars | nindent 8 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + mountPath: /opt/bitnami/airflow/dags + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + mountPath: /opt/bitnami/airflow/airflow.cfg + subPath: airflow.cfg + {{- end }} + resources: +{{ toYaml .Values.resources | indent 10 }} + volumes: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + configMap: + name: {{ template "airflow.dagFilesConfigMap" . }} + {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + configMap: + name: {{ .Values.airflow.airflowConfigurationConfigMap }} + {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/ingress.yaml b/bitnami/airflow/templates/ingress.yaml new file mode 100644 index 000000000..d5cb78e46 --- /dev/null +++ b/bitnami/airflow/templates/ingress.yaml @@ -0,0 +1,33 @@ +{{- if .Values.ingress.enabled }} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "airflow.fullname" . }} + labels: + app: "{{ template "airflow.fullname" . }}" + chart: "{{ template "airflow.chart" . }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} + annotations: + {{- if .Values.ingress.certManager }} + kubernetes.io/tls-acme: "true" + {{- end }} + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + - path: {{ default "/" .path }} + backend: + serviceName: "{{ template "airflow.fullname" $ }}" + servicePort: http + {{- end }} + {{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} + {{- end }} +{{- end }} diff --git a/bitnami/airflow/templates/secret-externaldb.yaml b/bitnami/airflow/templates/secret-externaldb.yaml new file mode 100644 index 000000000..cf5839870 --- /dev/null +++ b/bitnami/airflow/templates/secret-externaldb.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.postgresql.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: + db-password: {{ .Values.externalDatabase.password | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/secret-redis.yaml b/bitnami/airflow/templates/secret-redis.yaml new file mode 100644 index 000000000..0b5b42e72 --- /dev/null +++ b/bitnami/airflow/templates/secret-redis.yaml @@ -0,0 +1,14 @@ +{{- if not .Values.redis.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" .Release.Name "externalredis" }} + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: + redis-password: {{ .Values.externalRedis.password | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/secrets.yaml b/bitnami/airflow/templates/secrets.yaml new file mode 100644 index 000000000..632dbb224 --- /dev/null +++ b/bitnami/airflow/templates/secrets.yaml @@ -0,0 +1,23 @@ +{{- if (not .Values.airflow.auth.existingSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "airflow.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +type: Opaque +data: + {{- if .Values.airflow.auth.airflowPassword }} + airflow-password: {{ .Values.airflow.auth.airflowPassword | b64enc | quote }} + {{ else }} + airflow-password: {{ randAlphaNum 10 | b64enc | quote }} + {{- end }} + {{- if .Values.airflow.auth.fernetKey }} + airflow-fernetKey: {{ .Values.airflow.auth.fernetKey | b64enc | quote }} + {{ else }} + airflow-fernetKey: {{ randAlphaNum 32 | b64enc | b64enc | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml new file mode 100644 index 000000000..2fb5e43d3 --- /dev/null +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -0,0 +1,192 @@ +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: {{ template "airflow.fullname" . }}-worker + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + component: worker +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: worker + serviceName: {{ template "airflow.fullname" . }}-headless + replicas: {{ .Values.airflow.worker.replicas }} + updateStrategy: + type: {{ .Values.updateStrategy | quote }} + {{- if (eq "Recreate" .Values.updateStrategy) }} + rollingUpdate: null + {{- else if .Values.rollingUpdatePartition }} + rollingUpdate: + partition: {{ .Values.rollingUpdatePartition }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: worker + spec: +{{- include "airflow.imagePullSecrets" . | indent 6 }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + initContainers: + - name: git-clone-repository + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + volumeMounts: + - name: custom-dag-files + mountPath: /dags + {{- end }} + containers: + - name: airflow-worker + image: "{{ template "airflow.workerImage" . }}" + imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}" + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + {{- if .Values.workerImage.debug }} + - name: BASH_DEBUG + value: "1" + - name: NAMI_DEBUG + value: "1" + - name: NAMI_LOG_LEVEL + value: "trace8" + {{- end }} + - name: AIRFLOW_DATABASE_NAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.user | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "airflow.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: AIRFLOW_DATABASE_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "airflow.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: AIRFLOW_DATABASE_PORT_NUMBER + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port | quote }} + {{- end }} + - name: REDIS_HOST + {{- if .Values.redis.enabled }} + value: {{ template "airflow.redis.fullname" . }} + {{- else }} + value: {{ .Values.externalRedis.host | quote }} + {{- end }} + - name: REDIS_PORT_NUMBER + {{- if .Values.redis.enabled }} + value: "6379" + {{- else }} + value: {{ .Values.externalRedis.port | quote }} + {{- end }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.redis.enabled }} + name: {{ template "airflow.redis.secretName" . }} + key: redis-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externalredis" }} + key: redis-password + {{- end }} + - name: AIRFLOW_EXECUTOR + value: "CeleryExecutor" + - name: AIRFLOW_FERNET_KEY + valueFrom: + secretKeyRef: + name: {{ if .Values.airflow.auth.existingSecret }}{{ .Values.airflow.auth.existingSecret }}{{ else }}{{ template "airflow.fullname" . }}{{ end }} + key: airflow-fernetKey + - name: AIRFLOW_WEBSERVER_HOST + value: {{ template "airflow.fullname" . }} + {{- if .Values.airflow.extraEnvVars }} + {{ toYaml .Values.airflow.extraEnvVars | nindent 8 }} + {{- end }} + ports: + - name: worker + containerPort: {{ .Values.airflow.worker.port }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: worker + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: worker + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + mountPath: /opt/bitnami/airflow/dags + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + mountPath: /opt/bitnami/airflow/airflow.cfg + subPath: airflow.cfg + {{- end }} + volumes: + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: custom-dag-files + {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + configMap: + name: {{ template "airflow.dagFilesConfigMap" . }} + {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} + emptyDir: {} + {{- end }} + {{- end }} + {{- if .Values.airflow.airflowConfigurationConfigMap }} + - name: custom-configuration-file + configMap: + name: {{ .Values.airflow.airflowConfigurationConfigMap }} + {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/svc-headless-worker.yaml b/bitnami/airflow/templates/svc-headless-worker.yaml new file mode 100644 index 000000000..e104c84e2 --- /dev/null +++ b/bitnami/airflow/templates/svc-headless-worker.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "airflow.fullname" . }}-headless + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: airflow-worker + port: 8793 + targetPort: airflow-worker + selector: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: worker diff --git a/bitnami/airflow/templates/svc.yaml b/bitnami/airflow/templates/svc.yaml new file mode 100644 index 000000000..3ae900b8b --- /dev/null +++ b/bitnami/airflow/templates/svc.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "airflow.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + {{- range $key, $value := .Values.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if eq .Values.service.type "LoadBalancer" }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- end }} + ports: + - name: http + port: 8080 + {{- if and .Values.service.nodePort (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: http + selector: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: web diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml new file mode 100644 index 000000000..e757d6e4e --- /dev/null +++ b/bitnami/airflow/values.yaml @@ -0,0 +1,315 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName + +## Bitnami Airflow image version +## ref: https://hub.docker.com/r/bitnami/airflow/tags/ +## +image: + registry: docker.io + repository: bitnami/airflow + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami Airflow Scheduler image version +## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/ +## +schedulerImage: + registry: docker.io + repository: bitnami/airflow-scheduler + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami Airflow Worker image version +## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/ +## +workerImage: + registry: docker.io + repository: bitnami/airflow-worker + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami git image version +## ref: https://hub.docker.com/r/bitnami/git/tags/ +## +git: + registry: docker.io + repository: bitnami/git + tag: latest + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Airflow Components configuration +## +airflow: + ## Name of an existing config map containing the Airflow config file + ## + # airflowConfigurationConfigMap: + ## Name of an existing config map containing all the DAGs files you want to load in Airflow. + ## + # dagsConfigMap: + ## Airflow generic configuration + ## + loadExamples: true + ## Enable in order to download DAG files from git repository. + ## + cloneDagFilesFromGit: + enabled: false + repository: + revision: + ## URL used to access to airflow web ui + ## + # baseUrl: + ## Airflow worker component configuration + worker: + port: 8793 + replicas: 2 + + auth: + ## Specify username to acccess web UI + ## + airflowUsername: user + ## Specify password to acccess web UI + ## + # airflowPassword: + ## Specify a fernet key to secure airflow connections + ## More info at: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html + ## + # fernetKey: + ## Use existing secret (ignores previous airflowPassword and fernetKey) + ## + # existingSecret: + + ## All the parameters from the configuration file can be overwritten by using environment variables with this format: + ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html + ## + # extraEnvVars: + # - name: AIRFLOW__SMTP__SMTP_HOST + # value: "localhost" + # - name: AIRFLOW__SMTP__SMTP_PORT + # value: "25" + # - name: AIRFLOW__SMTP__USER + # value: "" + +## Kubernetes Security Context +## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 8080 + + ## Specify the NodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Use loadBalancerIP to request a specific static IP, + # loadBalancerIP: + + ## Service annotations done as key:value pairs + annotations: + +## Configure the ingress resource that allows you to access the +## Airflow installation. Set up the URL +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## +ingress: + ## Set to true to enable ingress record generation + enabled: false + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## Ingress annotations done as key:value pairs + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + annotations: + # kubernetes.io/ingress.class: nginx + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: airflow.local + path: / + + ## The tls configuration for the ingress + ## see: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls + tls: + - hosts: + - airflow.local + secretName: airflow.local-tls + + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using cert-manager, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: airflow.local-tls + # key: + # certificate: + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: +# limits: +# cpu: 200m +# memory: 1Gi +# requests: +# memory: 256Mi +# cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## +## PostgreSQL chart configuration +## +## https://github.com/helm/charts/blob/master/stable/postgresql/values.yaml +## +postgresql: + enabled: true + postgresqlUsername: bn_airflow + #postgresqlPassword: + postgresqlDatabase: bitnami_airflow + +externalDatabase: + ## All of these values are only used when postgresql.enabled is set to false + ## Database host + host: localhost + ## non-root Username for Airflow Database + user: bn_airflow + ## Database password + password: "" + ## Database name + database: bitnami_airflow + ## Database port number + port: 5432 + +## +## Redis chart configuration +## +## https://github.com/helm/charts/blob/master/stable/redis/values.yaml +## +redis: + enabled: true + +externalRedis: + ## All of these values are only used when redis.enabled is set to false + ## Redis host + host: localhost + ## Redis port number + port: 6379 + ## Redis password + password: "" \ No newline at end of file From 77c87c49376b68b0aa70a916d7a722c1779e7cc0 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Thu, 2 May 2019 11:45:38 +0200 Subject: [PATCH 02/11] Update NOTES.txt and other fixes --- bitnami/airflow/README.md | 3 +- bitnami/airflow/requirements.lock | 8 +- bitnami/airflow/requirements.yaml | 2 +- bitnami/airflow/templates/NOTES.txt | 68 +++- bitnami/airflow/templates/deployment-web.yaml | 2 +- ...t-redis.yaml => secret-externalredis.yaml} | 0 bitnami/airflow/templates/secrets.yaml | 8 +- .../templates/svc-headless-worker.yaml | 4 +- bitnami/airflow/values-production.yaml | 323 ++++++++++++++++++ bitnami/airflow/values.yaml | 12 +- 10 files changed, 411 insertions(+), 19 deletions(-) rename bitnami/airflow/templates/{secret-redis.yaml => secret-externalredis.yaml} (100%) create mode 100644 bitnami/airflow/values-production.yaml diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 112415516..76f3a1033 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -80,9 +80,10 @@ The following tables lists the configurable parameters of the Kafka chart and th | `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from | `nil` | | `airflow.cloneDagFilesFromGit.revision` | Revision from repository to checkout | `nil` | -| `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | +| `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | +| `airflow.auth.forcePassword` | Force users to specify a password | `false` | | `airflow.auth.airflowUsername` | Username to access web UI | `user` | | `airflow.auth.airflowPassword` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | diff --git a/bitnami/airflow/requirements.lock b/bitnami/airflow/requirements.lock index 69a8a58b1..66e273cf6 100644 --- a/bitnami/airflow/requirements.lock +++ b/bitnami/airflow/requirements.lock @@ -1,9 +1,9 @@ dependencies: - name: postgresql repository: https://charts.bitnami.com/bitnami - version: 3.18.2 + version: 3.18.3 - name: redis repository: https://charts.bitnami.com/bitnami - version: 6.4.5 -digest: sha256:f8531d43c9fce57c6c4c396d76bc752cc0065783bb18cba3b6bae50558e1a473 -generated: 2019-04-23T11:12:57.062256486+02:00 + version: 7.0.1 +digest: sha256:cf902ebdc6a871b1414ad64c95abb2fecf188b6cb0209bb91995620f84ca35ca +generated: 2019-05-02T11:16:02.72367469+02:00 diff --git a/bitnami/airflow/requirements.yaml b/bitnami/airflow/requirements.yaml index 0a819c03a..0a41b4323 100644 --- a/bitnami/airflow/requirements.yaml +++ b/bitnami/airflow/requirements.yaml @@ -4,6 +4,6 @@ dependencies: repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: redis - version: 6.x.x + version: 7.x.x repository: https://charts.bitnami.com/bitnami condition: redis.enabled diff --git a/bitnami/airflow/templates/NOTES.txt b/bitnami/airflow/templates/NOTES.txt index 5c0323989..ee4d54101 100644 --- a/bitnami/airflow/templates/NOTES.txt +++ b/bitnami/airflow/templates/NOTES.txt @@ -1,22 +1,78 @@ +{{- if not .Values.airflow.baseUrl -}} +############################################################################### +### ERROR: You did not provide an external URL in your 'helm install' call ### +############################################################################### + +This deployment will be incomplete until you configure Airflow with a resolvable +host. To configure Airflow with the URL of your service: + +{{- if .Values.ingress.enabled }} + +1. Get the Airflow URL indicated on the Ingress Rule and associate it to your cluster external IP: + + export CLUSTER_IP=$(minikube ip) # On Minikube. Use: `kubectl cluster-info` on others K8s clusters + export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath='{.spec.rules[0].host}') + echo "Airflow URL: http://$HOSTNAME/" + echo "$CLUSTER_IP $HOSTNAME" | sudo tee -a /etc/hosts + +{{- else }} 1. Get the Airflow URL by running: -{{- if eq .Values.service.type "ClusterIP" }} + {{- if contains "NodePort" .Values.service.type }} + + export APP_HOST=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export APP_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }}) + + {{- else if contains "ClusterIP" .Values.service.type }} + + export APP_HOST=127.0.0.1 + export APP_PORT=8080 + + {{- else if contains "LoadBalancer" .Values.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "airflow.fullname" . }}' + + export APP_HOST=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}") + export APP_PORT=80 + {{- end }} + export APP_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) + export APP_DATABASE_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.postgresql.fullname" . }} -o jsonpath="{.data.postgresql-password}" | base64 --decode) + export APP_REDIS_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.redis.secretName" . }} -o jsonpath="{.data.redis-password}" | base64 --decode) + +{{- end }} +2. Complete your Airflow deployment by running: + + helm upgrade {{ .Release.Name }} bitnami/{{ .Chart.Name }} \ + --set service.type={{ .Values.service.type }},airflow.baseUrl=http://$APP_HOST:$APP_PORT,airflow.auth.airflowPassword=$APP_PASSWORD,postgresql.postgresqlPassword=$APP_DATABASE_PASSWORD,redis.password=$APP_REDIS_PASSWORD{{- if .Values.global }}{{- if .Values.global.imagePullSecrets }},global.imagePullSecrets={{ .Values.global.imagePullSecrets }}{{- end }}{{- end }} + +{{- else -}} + +1. Get the Airflow URL by running: + +{{- if .Values.ingress.enabled }} + + echo URL : {{ .Values.airflow.baseUrl }} + +{{- else if eq .Values.service.type "ClusterIP" }} echo URL : http://127.0.0.1:{{ default "8080" .Values.service.port }} kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "airflow.fullname" . }} {{ default "8080" .Values.service.port }}:8080 -{{- else if eq .Values.service.type "NodePort" }} - export APP_BASE_URL=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") +{{- else if eq .Values.servicePort "nodePort" }} + export APP_HOST=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") export APP_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }}) - echo URL : $APP_BASE_URL + echo URL : http://$APP_HOST:$APP_PORT {{- else }} - echo URL : http://{{ .Values.airflow.baseUrl }} + echo URL : {{ .Values.airflow.baseUrl }} {{- end }} 2. Get your Airflow login credentials by running: echo User: {{ .Values.airflow.auth.airflowUsername }} - echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) \ No newline at end of file + echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) + + {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index 5d1a8304b..8c9bc3c06 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -141,7 +141,7 @@ spec: - name: AIRFLOW_WEBSERVER_HOST value: {{ template "airflow.fullname" . }} {{- if .Values.airflow.baseUrl }} - - name: AIRFLOW_BASE_URl + - name: AIRFLOW_BASE_URL value: {{ .Values.airflow.baseUrl }} {{- end }} - name: AIRFLOW_LOAD_EXAMPLES diff --git a/bitnami/airflow/templates/secret-redis.yaml b/bitnami/airflow/templates/secret-externalredis.yaml similarity index 100% rename from bitnami/airflow/templates/secret-redis.yaml rename to bitnami/airflow/templates/secret-externalredis.yaml diff --git a/bitnami/airflow/templates/secrets.yaml b/bitnami/airflow/templates/secrets.yaml index 632dbb224..c7336f38d 100644 --- a/bitnami/airflow/templates/secrets.yaml +++ b/bitnami/airflow/templates/secrets.yaml @@ -12,12 +12,16 @@ type: Opaque data: {{- if .Values.airflow.auth.airflowPassword }} airflow-password: {{ .Values.airflow.auth.airflowPassword | b64enc | quote }} - {{ else }} + {{ else if (not .Values.airflow.auth.forcePassword) }} airflow-password: {{ randAlphaNum 10 | b64enc | quote }} + {{ else }} + airflow-password: {{ required "An Airflow Password is required!" .Values.airflow.auth.airflowPassword }} {{- end }} {{- if .Values.airflow.auth.fernetKey }} airflow-fernetKey: {{ .Values.airflow.auth.fernetKey | b64enc | quote }} - {{ else }} + {{ else if (not .Values.airflow.auth.forcePassword) }} airflow-fernetKey: {{ randAlphaNum 32 | b64enc | b64enc | quote }} + {{ else }} + airflow-fernetKey: {{ required "An Airflow Fernet Key is required!" .Values.airflow.auth.fernetKey }} {{- end }} {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/svc-headless-worker.yaml b/bitnami/airflow/templates/svc-headless-worker.yaml index e104c84e2..b93b24a36 100644 --- a/bitnami/airflow/templates/svc-headless-worker.yaml +++ b/bitnami/airflow/templates/svc-headless-worker.yaml @@ -11,9 +11,9 @@ spec: type: ClusterIP clusterIP: None ports: - - name: airflow-worker + - name: worker port: 8793 - targetPort: airflow-worker + targetPort: worker selector: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml new file mode 100644 index 000000000..1cd85eb72 --- /dev/null +++ b/bitnami/airflow/values-production.yaml @@ -0,0 +1,323 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName + +## Bitnami Airflow image version +## ref: https://hub.docker.com/r/bitnami/airflow/tags/ +## +image: + registry: docker.io + repository: bitnami/airflow + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami Airflow Scheduler image version +## ref: https://hub.docker.com/r/bitnami/airflow-scheduler/tags/ +## +schedulerImage: + registry: docker.io + repository: bitnami/airflow-scheduler + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami Airflow Worker image version +## ref: https://hub.docker.com/r/bitnami/airflow-worker/tags/ +## +workerImage: + registry: docker.io + repository: bitnami/airflow-worker + tag: 1.10.3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## Bitnami git image version +## ref: https://hub.docker.com/r/bitnami/git/tags/ +## +git: + registry: docker.io + repository: bitnami/git + tag: latest + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Airflow Components configuration +## +airflow: + ## Name of an existing config map containing the Airflow config file + ## + # airflowConfigurationConfigMap: + ## Name of an existing config map containing all the DAGs files you want to load in Airflow. + ## + # dagsConfigMap: + ## Airflow generic configuration + ## + loadExamples: false + ## Enable in order to download DAG files from git repository. + ## + cloneDagFilesFromGit: + enabled: false + repository: + revision: + ## URL used to access to airflow web ui + ## + baseUrl: http://airflow.local + ## Airflow worker component configuration + worker: + port: 8793 + replicas: 3 + + auth: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not forced, a random password will be generated. + ## + forcePassword: true + ## Specify username to acccess web UI + ## + airflowUsername: user + ## Specify password to acccess web UI + ## + # airflowPassword: + ## Specify a fernet key to secure airflow connections + ## Fernet key must be 32 url-safe base64-encoded bytes. + ## More info at: + ## - https://airflow.readthedocs.io/en/stable/howto/secure-connections.html + ## - https://bcb.github.io/airflow/fernet-key + ## + # fernetKey: + ## Use existing secret (ignores previous airflowPassword and fernetKey) + ## + # existingSecret: + + ## All the parameters from the configuration file can be overwritten by using environment variables with this format: + ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html + ## + # extraEnvVars: + # - name: AIRFLOW__SMTP__SMTP_HOST + # value: "localhost" + # - name: AIRFLOW__SMTP__SMTP_PORT + # value: "25" + # - name: AIRFLOW__SMTP__USER + # value: "" + +## Kubernetes Security Context +## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 8080 + + ## Specify the NodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Use loadBalancerIP to request a specific static IP, + # loadBalancerIP: + + ## Service annotations done as key:value pairs + annotations: + +## Configure the ingress resource that allows you to access the +## Airflow installation. Set up the URL +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## +ingress: + ## Set to true to enable ingress record generation + enabled: true + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## Ingress annotations done as key:value pairs + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + annotations: + # kubernetes.io/ingress.class: nginx + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: airflow.local + path: / + + ## The tls configuration for the ingress + ## see: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls + tls: + - hosts: + - airflow.local + secretName: airflow.local-tls + + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using cert-manager, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: airflow.local-tls + # key: + # certificate: + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: +# limits: +# cpu: 200m +# memory: 1Gi +# requests: +# memory: 256Mi +# cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +livenessProbe: + enabled: true + initialDelaySeconds: 180 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## +## PostgreSQL chart configuration +## +## https://github.com/helm/charts/blob/master/stable/postgresql/values.yaml +## +postgresql: + enabled: true + postgresqlUsername: bn_airflow + #postgresqlPassword: + postgresqlDatabase: bitnami_airflow + +externalDatabase: + ## All of these values are only used when postgresql.enabled is set to false + ## Database host + host: localhost + ## non-root Username for Airflow Database + user: bn_airflow + ## Database password + password: "" + ## Database name + database: bitnami_airflow + ## Database port number + port: 5432 + +## +## Redis chart configuration +## +## https://github.com/helm/charts/blob/master/stable/redis/values.yaml +## +redis: + enabled: true + # password: "" + +externalRedis: + ## All of these values are only used when redis.enabled is set to false + ## Redis host + host: localhost + ## Redis port number + port: 6379 + ## Redis password + password: "" \ No newline at end of file diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index e757d6e4e..0eb928074 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -128,9 +128,13 @@ airflow: ## Airflow worker component configuration worker: port: 8793 - replicas: 2 + replicas: 1 auth: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not forced, a random password will be generated. + ## + forcePassword: false ## Specify username to acccess web UI ## airflowUsername: user @@ -138,7 +142,10 @@ airflow: ## # airflowPassword: ## Specify a fernet key to secure airflow connections - ## More info at: https://airflow.readthedocs.io/en/stable/howto/secure-connections.html + ## Fernet key must be 32 url-safe base64-encoded bytes. + ## More info at: + ## - https://airflow.readthedocs.io/en/stable/howto/secure-connections.html + ## - https://bcb.github.io/airflow/fernet-key ## # fernetKey: ## Use existing secret (ignores previous airflowPassword and fernetKey) @@ -304,6 +311,7 @@ externalDatabase: ## redis: enabled: true + # password: "" externalRedis: ## All of these values are only used when redis.enabled is set to false From 811b29c17046d97490eafe4beaf8f57a42d85abe Mon Sep 17 00:00:00 2001 From: tompizmor Date: Thu, 2 May 2019 12:48:46 +0200 Subject: [PATCH 03/11] Add metrics exporter --- bitnami/airflow/README.md | 189 +++++++++--------- bitnami/airflow/templates/_helpers.tpl | 23 +++ .../templates/deployment-scheduler.yaml | 6 +- bitnami/airflow/templates/deployment-web.yaml | 6 +- .../airflow/templates/metrics-deployment.yaml | 73 +++++++ bitnami/airflow/templates/metrics-svc.yaml | 22 ++ .../airflow/templates/statefulset-worker.yaml | 6 +- .../templates/svc-headless-worker.yaml | 2 +- bitnami/airflow/templates/svc.yaml | 2 +- bitnami/airflow/values-production.yaml | 36 +++- bitnami/airflow/values.yaml | 36 +++- 11 files changed, 297 insertions(+), 104 deletions(-) create mode 100644 bitnami/airflow/templates/metrics-deployment.yaml create mode 100644 bitnami/airflow/templates/metrics-svc.yaml diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 76f3a1033..3dacc4545 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -45,97 +45,104 @@ The command removes all the Kubernetes components associated with the chart and The following tables lists the configurable parameters of the Kafka chart and their default values. -| Parameter | Description | Default | -| ----------------------------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------- | -| `global.imageRegistry` | Global Docker image registry | `nil` | -| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `image.registry` | Airflow image registry | `docker.io` | -| `image.repository` | Airflow image name | `bitnami/airflow` | -| `image.tag` | Airflow image tag | `{VERSION}` | -| `image.pullPolicy` | Airflow image pull policy | `Always` | -| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `image.debug` | Specify if debug values should be set | `false` | -| `schedulerImage.registry` | Airflow Scheduler image registry | `docker.io` | -| `schedulerImage.repository` | Airflow Scheduler image name | `bitnami/airflow-shceduler` | -| `schedulerImage.tag` | Airflow Scheduler image tag | `{VERSION}` | -| `schedulerImage.pullPolicy` | Airflow Scheduler image pull policy | `Always` | -| `schedulerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `schedulerImage.debug` | Specify if debug values should be set | `false` | -| `workerImage.registry` | Airflow Worker image registry | `docker.io` | -| `workerImage.repository` | Airflow Worker image name | `bitnami/airflow-worker` | -| `workerImage.tag` | Airflow Worker image tag | `{VERSION}` | -| `workerImage.pullPolicy` | Airflow Worker image pull policy | `Always` | -| `workerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `workerImage.debug` | Specify if debug values should be set | `false` | -| `gitImage.registry` | Git image registry | `docker.io` | -| `gitImage.repository` | Git image name | `bitnami/git` | -| `gitImage.tag` | Git image tag | `{VERSION}` | -| `gitImage.pullPolicy` | Git image pull policy | `Always` | -| `gitImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | -| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | -| `rollingUpdatePartition` | Partition update strategy | `nil` | -| `airflow.airflowConfigurationConfigMap` | Name of an existing config map containing the Airflow config file | `nil` | -| `airflow.dagsConfigMap` | Name of an existing config map containing all the DAGs files you want to load in Airflow. | `nil` | -| `airflow.loadExamples` | Switch to load some Airflow examples | `true` | -| `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | -| `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from | `nil` | -| `airflow.cloneDagFilesFromGit.revision` | Revision from repository to checkout | `nil` | -| `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | -| `airflow.worker.port` | Airflow Worker port | `8793` | -| `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | -| `airflow.auth.forcePassword` | Force users to specify a password | `false` | -| `airflow.auth.airflowUsername` | Username to access web UI | `user` | -| `airflow.auth.airflowPassword` | Password to access web UI | `nil` | -| `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | -| `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | -| `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | -| `securityContext.enabled` | Enable security context | `true` | -| `securityContext.fsGroup` | Group ID for the container | `1001` | -| `securityContext.runAsUser` | User ID for the container | `1001` | -| `service.type` | Kubernetes Service type | `ClusterIP` | -| `service.port` | Airflow Web port | `8080` | -| `service.nodePort` | Kubernetes Service nodePort | `nil` | -| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | -| `service.annotations` | Service annotations | `` | -| `ingress.enabled` | Enable ingress controller resource | `false` | -| `ingress.certManager` | Add annotations for cert-manager | `false` | -| `ingress.annotations` | Ingress annotations | `[]` | -| `ingress.hosts[0].name` | Hostname to your Wordpress installation | `airflow.local` | -| `ingress.hosts[0].path` | Path within the url structure | `/` | -| `ingress.tls[0].hosts[0]` | TLS hosts | `airflow.local` | -| `ingress.tls[0].secretName` | TLS Secret (certificates) | `airflow.local-tls` | -| `ingress.secrets[0].name` | TLS Secret Name | `nil` | -| `ingress.secrets[0].certificate` | TLS Secret Certificate | `nil` | -| `ingress.secrets[0].key` | TLS Secret Key | `nil` | -| `nodeSelector` | Node labels for pod assignment | `{}` | -| `tolerations` | Toleration labels for pod assignment | `[]` | -| `affinity` | Map of node/pod affinities | `{}` | -| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | -| `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | -| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 180 | -| `livenessProbe.periodSeconds` | How often to perform the probe | 20 | -| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | -| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | -| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | -| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | -| `readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | -| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | -| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | -| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | -| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | -| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | -| `postgresql.postgresqlUsername` | Airflow Postgresql username | `bn_airflow` | -| `postgresql.postgresqlPassword` | Airflow Postgresql password | `nil` | -| `postgresql.postgresqlDatabase` | Airflow Postgresql database | `bitnami_airflow` | -| `externalDatabase.host` | External PostgreSQL host | `nil` | -| `externalDatabase.user` | External PostgreSQL user | `nil` | -| `externalDatabase.password` | External PostgreSQL password | `nil` | -| `externalDatabase.database` | External PostgreSQL database name | `nil` | -| `externalDatabase.port` | External PostgreSQL port | `nil` | -| `redis.enabled` | Switch to enable or disable the Redis helm chart | `true` | -| `externalRedis.host` | External Redis host | `nil` | -| `externalRedis.port` | External Redis port | `nil` | -| `externalRedis.password` | External Redis password | `nil` | +| Parameter | Description | Default | +| ----------------------------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------------ | +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.registry` | Airflow image registry | `docker.io` | +| `image.repository` | Airflow image name | `bitnami/airflow` | +| `image.tag` | Airflow image tag | `{VERSION}` | +| `image.pullPolicy` | Airflow image pull policy | `Always` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `schedulerImage.registry` | Airflow Scheduler image registry | `docker.io` | +| `schedulerImage.repository` | Airflow Scheduler image name | `bitnami/airflow-shceduler` | +| `schedulerImage.tag` | Airflow Scheduler image tag | `{VERSION}` | +| `schedulerImage.pullPolicy` | Airflow Scheduler image pull policy | `Always` | +| `schedulerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `schedulerImage.debug` | Specify if debug values should be set | `false` | +| `workerImage.registry` | Airflow Worker image registry | `docker.io` | +| `workerImage.repository` | Airflow Worker image name | `bitnami/airflow-worker` | +| `workerImage.tag` | Airflow Worker image tag | `{VERSION}` | +| `workerImage.pullPolicy` | Airflow Worker image pull policy | `Always` | +| `workerImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `workerImage.debug` | Specify if debug values should be set | `false` | +| `gitImage.registry` | Git image registry | `docker.io` | +| `gitImage.repository` | Git image name | `bitnami/git` | +| `gitImage.tag` | Git image tag | `{VERSION}` | +| `gitImage.pullPolicy` | Git image pull policy | `Always` | +| `gitImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `airflow.airflowConfigurationConfigMap` | Name of an existing config map containing the Airflow config file | `nil` | +| `airflow.dagsConfigMap` | Name of an existing config map containing all the DAGs files you want to load in Airflow. | `nil` | +| `airflow.loadExamples` | Switch to load some Airflow examples | `true` | +| `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | +| `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from | `nil` | +| `airflow.cloneDagFilesFromGit.revision` | Revision from repository to checkout | `nil` | +| `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | +| `airflow.worker.port` | Airflow Worker port | `8793` | +| `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | +| `airflow.auth.forcePassword` | Force users to specify a password | `false` | +| `airflow.auth.airflowUsername` | Username to access web UI | `user` | +| `airflow.auth.airflowPassword` | Password to access web UI | `nil` | +| `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | +| `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | +| `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the container | `1001` | +| `securityContext.runAsUser` | User ID for the container | `1001` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Airflow Web port | `8080` | +| `service.nodePort` | Kubernetes Service nodePort | `nil` | +| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | +| `service.annotations` | Service annotations | `` | +| `ingress.enabled` | Enable ingress controller resource | `false` | +| `ingress.certManager` | Add annotations for cert-manager | `false` | +| `ingress.annotations` | Ingress annotations | `[]` | +| `ingress.hosts[0].name` | Hostname to your Wordpress installation | `airflow.local` | +| `ingress.hosts[0].path` | Path within the url structure | `/` | +| `ingress.tls[0].hosts[0]` | TLS hosts | `airflow.local` | +| `ingress.tls[0].secretName` | TLS Secret (certificates) | `airflow.local-tls` | +| `ingress.secrets[0].name` | TLS Secret Name | `nil` | +| `ingress.secrets[0].certificate` | TLS Secret Certificate | `nil` | +| `ingress.secrets[0].key` | TLS Secret Key | `nil` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | +| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 180 | +| `livenessProbe.periodSeconds` | How often to perform the probe | 20 | +| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | +| `readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | +| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `postgresql.enabled` | Switch to enable or disable the PostgreSQL helm chart | `true` | +| `postgresql.postgresqlUsername` | Airflow Postgresql username | `bn_airflow` | +| `postgresql.postgresqlPassword` | Airflow Postgresql password | `nil` | +| `postgresql.postgresqlDatabase` | Airflow Postgresql database | `bitnami_airflow` | +| `externalDatabase.host` | External PostgreSQL host | `nil` | +| `externalDatabase.user` | External PostgreSQL user | `nil` | +| `externalDatabase.password` | External PostgreSQL password | `nil` | +| `externalDatabase.database` | External PostgreSQL database name | `nil` | +| `externalDatabase.port` | External PostgreSQL port | `nil` | +| `redis.enabled` | Switch to enable or disable the Redis helm chart | `true` | +| `externalRedis.host` | External Redis host | `nil` | +| `externalRedis.port` | External Redis port | `nil` | +| `externalRedis.password` | External Redis password | `nil` | +| `metrics.enabled` | Start a side-car prometheus exporter | `false` | +| `metrics.image.registry` | Airflow exporter image registry | `docker.io` | +| `metrics.image.repository` | Airflow exporter image name | `pbweb/airflow-prometheus-exporter` | +| `metrics.image.tag` | Airflow exporter image tag | `latest` | +| `metrics.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `metrics.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.podAnnotations` | Additional annotations for Metrics exporter | `{prometheus.io/scrape: "true", prometheus.io/port: "9112"}` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index b30984243..2bd71b7f7 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -123,6 +123,29 @@ Also, we can't use a single if because lazy evaluation is not an option {{- end -}} {{- end -}} +{{/* +Return the proper Airflow Metrics image name +*/}} +{{- define "airflow.metrics.image" -}} +{{- $registryName := .Values.metrics.image.registry -}} +{{- $repositoryName := .Values.metrics.image.repository -}} +{{- $tag := .Values.metrics.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + {{/* Return the proper Docker Image Registry Secret Names */}} diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index bc5a701ee..f659b9edd 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -7,13 +7,13 @@ metadata: helm.sh/chart: {{ include "airflow.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} - component: scheduler + app.kubernetes.io/component: scheduler spec: selector: matchLabels: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: scheduler + app.kubernetes.io/component: scheduler replicas: {{ .Values.replicaCount }} template: metadata: @@ -21,7 +21,7 @@ spec: app.kubernetes.io/name: {{ include "airflow.name" . }} helm.sh/chart: {{ include "airflow.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: scheduler + app.kubernetes.io/component: scheduler {{- if .Values.podAnnotations }} annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index 8c9bc3c06..be25e27cc 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -7,13 +7,13 @@ metadata: helm.sh/chart: {{ include "airflow.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} - component: web + app.kubernetes.io/component: web spec: selector: matchLabels: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: web + app.kubernetes.io/component: web replicas: {{ .Values.replicaCount }} template: metadata: @@ -21,7 +21,7 @@ spec: app.kubernetes.io/name: {{ include "airflow.name" . }} helm.sh/chart: {{ include "airflow.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: web + app.kubernetes.io/component: web {{- if .Values.podAnnotations }} annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} diff --git a/bitnami/airflow/templates/metrics-deployment.yaml b/bitnami/airflow/templates/metrics-deployment.yaml new file mode 100644 index 000000000..81d6116c5 --- /dev/null +++ b/bitnami/airflow/templates/metrics-deployment.yaml @@ -0,0 +1,73 @@ +{{- if .Values.metrics.enabled }} +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ template "airflow.fullname" . }}-exporter + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + annotations: +{{ toYaml .Values.metrics.podAnnotations | indent 8 }} + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/component: metrics + app.kubernetes.io/instance: {{ .Release.Name }} + spec: +{{- include "airflow.imagePullSecrets" . | indent 6 }} + containers: + - image: {{ template "airflow.metrics.image" . }} + name: airflow-exporter + env: + - name: AIRFLOW_PROMETHEUS_DATABASE_BACKEND + value: "postgres" + - name: AIRFLOW_PROMETHEUS_DATABASE_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "airflow.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} + - name: AIRFLOW_PROMETHEUS_DATABASE_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port | quote }} + {{- end }} + - name: AIRFLOW_PROMETHEUS_DATABASE_USER + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.user | quote }} + {{- end }} + - name: AIRFLOW_PROMETHEUS_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "airflow.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ printf "%s-%s" .Release.Name "externaldb" }} + key: db-password + {{- end }} + - name: AIRFLOW_PROMETHEUS_DATABASE_NAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} + ports: + - name: metrics + containerPort: 9112 + resources: +{{ toYaml .Values.metrics.resources | indent 10 }} +{{- end }} diff --git a/bitnami/airflow/templates/metrics-svc.yaml b/bitnami/airflow/templates/metrics-svc.yaml new file mode 100644 index 000000000..c19aa5600 --- /dev/null +++ b/bitnami/airflow/templates/metrics-svc.yaml @@ -0,0 +1,22 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "airflow.fullname" . }}-exporter + labels: + app.kubernetes.io/name: {{ include "airflow.name" . }} + helm.sh/chart: {{ include "airflow.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: metrics +spec: + type: ClusterIP + ports: + - name: metrics + port: 9112 + targetPort: metrics + selector: + app.kubernetes.io/name: {{ include "airflow.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: metrics +{{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index 2fb5e43d3..c00f4c3f0 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -7,13 +7,13 @@ metadata: helm.sh/chart: {{ include "airflow.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} - component: worker + app.kubernetes.io/component: worker spec: selector: matchLabels: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: worker + app.kubernetes.io/component: worker serviceName: {{ template "airflow.fullname" . }}-headless replicas: {{ .Values.airflow.worker.replicas }} updateStrategy: @@ -29,7 +29,7 @@ spec: labels: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: worker + app.kubernetes.io/component: worker spec: {{- include "airflow.imagePullSecrets" . | indent 6 }} {{- if .Values.securityContext.enabled }} diff --git a/bitnami/airflow/templates/svc-headless-worker.yaml b/bitnami/airflow/templates/svc-headless-worker.yaml index b93b24a36..60443ad25 100644 --- a/bitnami/airflow/templates/svc-headless-worker.yaml +++ b/bitnami/airflow/templates/svc-headless-worker.yaml @@ -17,4 +17,4 @@ spec: selector: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: worker + app.kubernetes.io/component: worker diff --git a/bitnami/airflow/templates/svc.yaml b/bitnami/airflow/templates/svc.yaml index 3ae900b8b..63b7279de 100644 --- a/bitnami/airflow/templates/svc.yaml +++ b/bitnami/airflow/templates/svc.yaml @@ -28,4 +28,4 @@ spec: selector: app.kubernetes.io/name: {{ include "airflow.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - component: web + app.kubernetes.io/component: web diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index 1cd85eb72..163c505f4 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -320,4 +320,38 @@ externalRedis: ## Redis port number port: 6379 ## Redis password - password: "" \ No newline at end of file + password: "" + +## Prometheus Exporter / Metrics +## +metrics: + enabled: false + + image: + registry: docker.io + repository: pbweb/airflow-prometheus-exporter + tag: latest + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9112" + + ## Metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + # resources: {} + + ## Metrics exporter labels and tolerations for pod assignment + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + + ## Metrics exporter pod Annotation and Labels + # podAnnotations: {} + # podLabels: {} \ No newline at end of file diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 0eb928074..46d98a97e 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -320,4 +320,38 @@ externalRedis: ## Redis port number port: 6379 ## Redis password - password: "" \ No newline at end of file + password: "" + +## Prometheus Exporter / Metrics +## +metrics: + enabled: false + + image: + registry: docker.io + repository: pbweb/airflow-prometheus-exporter + tag: latest + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9112" + + ## Metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + # resources: {} + + ## Metrics exporter labels and tolerations for pod assignment + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + + ## Metrics exporter pod Annotation and Labels + # podAnnotations: {} + # podLabels: {} \ No newline at end of file From 34bd72591e15c7ad80a458ea53be1866672a365c Mon Sep 17 00:00:00 2001 From: tompizmor Date: Thu, 2 May 2019 13:00:02 +0200 Subject: [PATCH 04/11] Update imagePullSecrets helper --- bitnami/airflow/templates/_helpers.tpl | 29 ++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 2bd71b7f7..1b21498fb 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -146,6 +146,7 @@ Also, we can't use a single if because lazy evaluation is not an option {{- end -}} {{- end -}} + {{/* Return the proper Docker Image Registry Secret Names */}} @@ -161,17 +162,41 @@ imagePullSecrets: {{- range .Values.global.imagePullSecrets }} - name: {{ . }} {{- end }} -{{- else if or .Values.image.pullSecrets }} +{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.gitImage.pullSecrets .Values.metrics.image.pullSecrets }} imagePullSecrets: {{- range .Values.image.pullSecrets }} - name: {{ . }} {{- end }} +{{- range .Values.schedulerImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.workerImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.gitImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metrics.image.pullSecrets }} + - name: {{ . }} +{{- end }} {{- end -}} -{{- else if or .Values.image.pullSecrets }} +{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.gitImage.pullSecrets .Values.metrics.image.pullSecrets }} imagePullSecrets: {{- range .Values.image.pullSecrets }} - name: {{ . }} {{- end }} +{{- range .Values.schedulerImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.workerImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.gitImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metricsImage.pullSecrets }} + - name: {{ . }} +{{- end }} {{- end -}} {{- end -}} From 9e5b51cb110080a755e0cc2128d5ec378bca2b08 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Fri, 3 May 2019 10:31:33 +0200 Subject: [PATCH 05/11] Fix imagePullSecret template --- bitnami/airflow/templates/_helpers.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 1b21498fb..505ebf236 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -162,7 +162,7 @@ imagePullSecrets: {{- range .Values.global.imagePullSecrets }} - name: {{ . }} {{- end }} -{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.gitImage.pullSecrets .Values.metrics.image.pullSecrets }} +{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.git.pullSecrets .Values.metrics.image.pullSecrets }} imagePullSecrets: {{- range .Values.image.pullSecrets }} - name: {{ . }} @@ -173,14 +173,14 @@ imagePullSecrets: {{- range .Values.workerImage.pullSecrets }} - name: {{ . }} {{- end }} -{{- range .Values.gitImage.pullSecrets }} +{{- range .Values.git.pullSecrets }} - name: {{ . }} {{- end }} {{- range .Values.metrics.image.pullSecrets }} - name: {{ . }} {{- end }} {{- end -}} -{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.gitImage.pullSecrets .Values.metrics.image.pullSecrets }} +{{- else if or .Values.image.pullSecrets .Values.schedulerImage.pullSecrets .Values.workerImage.pullSecrets .Values.git.pullSecrets .Values.metrics.image.pullSecrets }} imagePullSecrets: {{- range .Values.image.pullSecrets }} - name: {{ . }} @@ -191,7 +191,7 @@ imagePullSecrets: {{- range .Values.workerImage.pullSecrets }} - name: {{ . }} {{- end }} -{{- range .Values.gitImage.pullSecrets }} +{{- range .Values.git.pullSecrets }} - name: {{ . }} {{- end }} {{- range .Values.metricsImage.pullSecrets }} From 5893c2bea2c5d0e7dc6f1377978f3ac2f7d7b5e0 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Fri, 3 May 2019 10:58:26 +0200 Subject: [PATCH 06/11] Fix imagePullSecret template for metrics image --- bitnami/airflow/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 505ebf236..ed6678b53 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -194,7 +194,7 @@ imagePullSecrets: {{- range .Values.git.pullSecrets }} - name: {{ . }} {{- end }} -{{- range .Values.metricsImage.pullSecrets }} +{{- range .Values.metrics.image.pullSecrets }} - name: {{ . }} {{- end }} {{- end -}} From fe9e6ee946bcb2713a01ac38f5ab7a65bb20728c Mon Sep 17 00:00:00 2001 From: tompizmor Date: Fri, 3 May 2019 13:55:32 +0200 Subject: [PATCH 07/11] Truncate several names to 63 chars instead of 24 --- bitnami/airflow/templates/_helpers.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index ed6678b53..80c5c29d6 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -205,7 +205,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "airflow.postgresql.fullname" -}} {{- $name := default "postgresql" .Values.postgresql.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} Create a default fully qualified redis name. @@ -213,7 +213,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "airflow.redis.fullname" -}} {{- $name := default "redis" .Values.redis.nameOverride -}} -{{- printf "%s-%s-master" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- printf "%s-%s-master" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} Create a template for the redis secret @@ -221,7 +221,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this */}} {{- define "airflow.redis.secretName" -}} {{- $name := default "redis" .Values.redis.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 24 | trimSuffix "-" -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} {{ template "airflow.dagFilesConfigMap" . }} From 6750e8940f4eaa3943edaac87de62fcb98afe395 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Mon, 6 May 2019 17:21:55 +0200 Subject: [PATCH 08/11] Add javsalgar feedback --- bitnami/airflow/README.md | 20 ++++++--- bitnami/airflow/templates/NOTES.txt | 4 +- bitnami/airflow/templates/_helpers.tpl | 12 ++--- bitnami/airflow/templates/configmap-dags.yaml | 4 +- .../templates/deployment-scheduler.yaml | 43 +++++++++++------- bitnami/airflow/templates/deployment-web.yaml | 45 ++++++++++++------- bitnami/airflow/templates/secrets.yaml | 6 +-- .../airflow/templates/statefulset-worker.yaml | 43 +++++++++++------- bitnami/airflow/values-production.yaml | 8 ++-- bitnami/airflow/values.yaml | 18 ++++---- 10 files changed, 125 insertions(+), 78 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 3dacc4545..eeac45eb1 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -74,7 +74,7 @@ The following tables lists the configurable parameters of the Kafka chart and th | `gitImage.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | | `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | | `rollingUpdatePartition` | Partition update strategy | `nil` | -| `airflow.airflowConfigurationConfigMap` | Name of an existing config map containing the Airflow config file | `nil` | +| `airflow.configurationConfigMap` | Name of an existing config map containing the Airflow config file | `nil` | | `airflow.dagsConfigMap` | Name of an existing config map containing all the DAGs files you want to load in Airflow. | `nil` | | `airflow.loadExamples` | Switch to load some Airflow examples | `true` | | `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | @@ -84,8 +84,8 @@ The following tables lists the configurable parameters of the Kafka chart and th | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | | `airflow.auth.forcePassword` | Force users to specify a password | `false` | -| `airflow.auth.airflowUsername` | Username to access web UI | `user` | -| `airflow.auth.airflowPassword` | Password to access web UI | `nil` | +| `airflow.auth.username` | Username to access web UI | `user` | +| `airflow.auth.password` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | @@ -148,8 +148,10 @@ Specify each parameter using the `--set key=value[,key=value]` argument to `helm ```console $ helm install --name my-release \ - --set airflow.auth.airflowUsername=my-user,airflow.auth.airflowPassword=my-passsword \ - bitnami/airflow + --set airflow.auth.username=my-user \ + --set airflow.auth.password=my-passsword \ + --set airflow.auth.fernetKey=my-fernet-key \ + bitnami/airflow ``` The above command sets the credentials to access the Airflow web UI. @@ -166,9 +168,15 @@ $ helm install --name my-release -f values.yaml bitnami/airflow The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means that Airflow does not persist anything. +## Generate a Fernet key + +A Fernet key is required in order to encrypt password within connections. The Fernet key must be a base64-encoded 32-byte key. + +Learn how to generate one [here](https://bcb.github.io/airflow/fernet-key) + ## Load DAG files -There are three different ways to load your custom DAG files into the Airflow chart: +There are three different ways to load your custom DAG files into the Airflow chart. All of them are compatible so you can use more than one at the same time. ### Option 1: Load locally from the `files` folder diff --git a/bitnami/airflow/templates/NOTES.txt b/bitnami/airflow/templates/NOTES.txt index ee4d54101..67023e306 100644 --- a/bitnami/airflow/templates/NOTES.txt +++ b/bitnami/airflow/templates/NOTES.txt @@ -44,7 +44,7 @@ host. To configure Airflow with the URL of your service: 2. Complete your Airflow deployment by running: helm upgrade {{ .Release.Name }} bitnami/{{ .Chart.Name }} \ - --set service.type={{ .Values.service.type }},airflow.baseUrl=http://$APP_HOST:$APP_PORT,airflow.auth.airflowPassword=$APP_PASSWORD,postgresql.postgresqlPassword=$APP_DATABASE_PASSWORD,redis.password=$APP_REDIS_PASSWORD{{- if .Values.global }}{{- if .Values.global.imagePullSecrets }},global.imagePullSecrets={{ .Values.global.imagePullSecrets }}{{- end }}{{- end }} + --set service.type={{ .Values.service.type }},airflow.baseUrl=http://$APP_HOST:$APP_PORT,airflow.auth.password=$APP_PASSWORD,postgresql.postgresqlPassword=$APP_DATABASE_PASSWORD,redis.password=$APP_REDIS_PASSWORD{{- if .Values.global }}{{- if .Values.global.imagePullSecrets }},global.imagePullSecrets={{ .Values.global.imagePullSecrets }}{{- end }}{{- end }} {{- else -}} @@ -72,7 +72,7 @@ host. To configure Airflow with the URL of your service: 2. Get your Airflow login credentials by running: - echo User: {{ .Values.airflow.auth.airflowUsername }} + echo User: {{ .Values.airflow.auth.username }} echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 80c5c29d6..095610006 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -224,14 +224,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{ template "airflow.dagFilesConfigMap" . }} + {{/* -Get the DAG files ConfigMap name. +Get the secret name */}} -{{- define "airflow.dagFilesConfigMap" -}} -{{- if .Values.airflow.dagsConfigMap -}} -{{- printf "%s" .Values.airflow.dagsConfigMap -}} +{{- define "airflow.secretName" -}} +{{- if .Values.airflow.auth.existingSecret -}} +{{- printf "%s" .Values.airflow.auth.existingSecret -}} {{- else -}} -{{- printf "%s-dag-files" (include "airflow.fullname" .) -}} +{{- printf "%s" (include "airflow.fullname" .) -}} {{- end -}} {{- end -}} \ No newline at end of file diff --git a/bitnami/airflow/templates/configmap-dags.yaml b/bitnami/airflow/templates/configmap-dags.yaml index 3dab10817..569377c32 100644 --- a/bitnami/airflow/templates/configmap-dags.yaml +++ b/bitnami/airflow/templates/configmap-dags.yaml @@ -1,4 +1,4 @@ -{{- if and (.Files.Glob "files/dags/*.py") (not .Values.airflow.dagsConfigMap) }} +{{- if .Files.Glob "files/dags/*.py" }} apiVersion: v1 kind: ConfigMap metadata: @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: -{{ (.Files.Glob "files/dags/*.{py}").AsConfig | indent 2 }} +{{ (.Files.Glob "files/dags/*.py").AsConfig | indent 2 }} {{ end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index f659b9edd..67a95fafc 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -45,14 +45,14 @@ spec: fsGroup: {{ .Values.securityContext.fsGroup }} runAsUser: {{ .Values.securityContext.runAsUser }} {{- end }} - {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} initContainers: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: custom-dag-files + - name: github-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -149,11 +149,19 @@ spec: readinessProbe: {{- end }} volumeMounts: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - mountPath: /opt/bitnami/airflow/dags + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files + mountPath: /opt/bitnami/airflow/dags/local {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + mountPath: /opt/bitnami/airflow/dags/external + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/github + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg @@ -161,17 +169,22 @@ spec: resources: {{ toYaml .Values.resources | indent 10 }} volumes: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files configMap: - name: {{ template "airflow.dagFilesConfigMap" . }} - {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} - emptyDir: {} - {{- end }} + name: {{ template "airflow.fullname" . }}-dag-files {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + configMap: + name: {{ .Values.airflow.dagsConfigMap }} + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + emptyDir: {} + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file configMap: - name: {{ .Values.airflow.airflowConfigurationConfigMap }} + name: {{ .Values.airflow.configurationConfigMap }} {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index be25e27cc..7a917f57b 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -45,14 +45,14 @@ spec: fsGroup: {{ .Values.securityContext.fsGroup }} runAsUser: {{ .Values.securityContext.runAsUser }} {{- end }} - {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} initContainers: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: custom-dag-files + - name: github-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -127,7 +127,7 @@ spec: - name: AIRFLOW_EXECUTOR value: "CeleryExecutor" - name: AIRFLOW_USERNAME - value: {{ .Values.airflow.auth.airflowUsername }} + value: {{ .Values.airflow.auth.username }} - name: AIRFLOW_PASSWORD valueFrom: secretKeyRef: @@ -179,11 +179,19 @@ spec: failureThreshold: {{ .Values.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - mountPath: /opt/bitnami/airflow/dags + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files + mountPath: /opt/bitnami/airflow/dags/local {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + mountPath: /opt/bitnami/airflow/dags/external + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/github + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg @@ -191,17 +199,22 @@ spec: resources: {{ toYaml .Values.resources | indent 10 }} volumes: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files configMap: - name: {{ template "airflow.dagFilesConfigMap" . }} - {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} - emptyDir: {} - {{- end }} + name: {{ template "airflow.fullname" . }}-dag-files {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + configMap: + name: {{ .Values.airflow.dagsConfigMap }} + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + emptyDir: {} + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file configMap: - name: {{ .Values.airflow.airflowConfigurationConfigMap }} + name: {{ .Values.airflow.configurationConfigMap }} {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/templates/secrets.yaml b/bitnami/airflow/templates/secrets.yaml index c7336f38d..d3678825a 100644 --- a/bitnami/airflow/templates/secrets.yaml +++ b/bitnami/airflow/templates/secrets.yaml @@ -10,12 +10,12 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} type: Opaque data: - {{- if .Values.airflow.auth.airflowPassword }} - airflow-password: {{ .Values.airflow.auth.airflowPassword | b64enc | quote }} + {{- if .Values.airflow.auth.password }} + airflow-password: {{ .Values.airflow.auth.password | b64enc | quote }} {{ else if (not .Values.airflow.auth.forcePassword) }} airflow-password: {{ randAlphaNum 10 | b64enc | quote }} {{ else }} - airflow-password: {{ required "An Airflow Password is required!" .Values.airflow.auth.airflowPassword }} + airflow-password: {{ required "An Airflow Password is required!" .Values.airflow.auth.password }} {{- end }} {{- if .Values.airflow.auth.fernetKey }} airflow-fernetKey: {{ .Values.airflow.auth.fernetKey | b64enc | quote }} diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index c00f4c3f0..7caed12c0 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -49,14 +49,14 @@ spec: affinity: {{ toYaml .Values.affinity | indent 8 }} {{- end }} - {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (not .Values.airflow.dagsConfigMap) }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} initContainers: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: custom-dag-files + - name: github-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -166,27 +166,40 @@ spec: failureThreshold: {{ .Values.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - mountPath: /opt/bitnami/airflow/dags + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files + mountPath: /opt/bitnami/airflow/dags/local {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + mountPath: /opt/bitnami/airflow/dags/external + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/github + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg {{- end }} volumes: - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: custom-dag-files - {{- if or (.Files.Glob "files/dags/*.py") .Values.airflow.dagsConfigMap }} + {{- if .Files.Glob "files/dags/*.py" }} + - name: local-dag-files configMap: - name: {{ template "airflow.dagFilesConfigMap" . }} - {{- else if .Values.airflow.cloneDagFilesFromGit.enabled }} - emptyDir: {} - {{- end }} + name: {{ template "airflow.fullname" . }}-dag-files {{- end }} - {{- if .Values.airflow.airflowConfigurationConfigMap }} + {{- if .Values.airflow.dagsConfigMap }} + - name: external-dag-files + configMap: + name: {{ .Values.airflow.dagsConfigMap }} + {{- end }} + {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} + - name: github-cloned-dag-files + emptyDir: {} + {{- end }} + {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file configMap: - name: {{ .Values.airflow.airflowConfigurationConfigMap }} + name: {{ .Values.airflow.configurationConfigMap }} {{- end }} \ No newline at end of file diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index 163c505f4..d3ca6d1b0 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -109,7 +109,7 @@ updateStrategy: RollingUpdate airflow: ## Name of an existing config map containing the Airflow config file ## - # airflowConfigurationConfigMap: + # configurationConfigMap: ## Name of an existing config map containing all the DAGs files you want to load in Airflow. ## # dagsConfigMap: @@ -137,10 +137,10 @@ airflow: forcePassword: true ## Specify username to acccess web UI ## - airflowUsername: user + username: user ## Specify password to acccess web UI ## - # airflowPassword: + # password: ## Specify a fernet key to secure airflow connections ## Fernet key must be 32 url-safe base64-encoded bytes. ## More info at: @@ -148,7 +148,7 @@ airflow: ## - https://bcb.github.io/airflow/fernet-key ## # fernetKey: - ## Use existing secret (ignores previous airflowPassword and fernetKey) + ## Use existing secret (ignores previous password and fernetKey) ## # existingSecret: diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 46d98a97e..fbee75473 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -109,19 +109,19 @@ updateStrategy: RollingUpdate airflow: ## Name of an existing config map containing the Airflow config file ## - # airflowConfigurationConfigMap: + # configurationConfigMap: ## Name of an existing config map containing all the DAGs files you want to load in Airflow. ## - # dagsConfigMap: + dagsConfigMap: patata ## Airflow generic configuration ## - loadExamples: true + loadExamples: false ## Enable in order to download DAG files from git repository. ## cloneDagFilesFromGit: - enabled: false - repository: - revision: + enabled: true + repository: https://github.com/tompizmor/sample-airflow-dag + revision: master ## URL used to access to airflow web ui ## # baseUrl: @@ -137,10 +137,10 @@ airflow: forcePassword: false ## Specify username to acccess web UI ## - airflowUsername: user + username: user ## Specify password to acccess web UI ## - # airflowPassword: + # password: ## Specify a fernet key to secure airflow connections ## Fernet key must be 32 url-safe base64-encoded bytes. ## More info at: @@ -148,7 +148,7 @@ airflow: ## - https://bcb.github.io/airflow/fernet-key ## # fernetKey: - ## Use existing secret (ignores previous airflowPassword and fernetKey) + ## Use existing secret (ignores previous password and fernetKey) ## # existingSecret: From aaa4e4688a5bfc9895da242f61df9354b8572d5d Mon Sep 17 00:00:00 2001 From: tompizmor Date: Tue, 7 May 2019 08:45:15 +0200 Subject: [PATCH 09/11] Delete extra blank line --- bitnami/airflow/templates/_helpers.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 095610006..a7aeff416 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -224,7 +224,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} {{- end -}} - {{/* Get the secret name */}} From edd83e9c2ab0b61164ed13a10e75e577224a2f68 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Tue, 7 May 2019 17:28:16 +0200 Subject: [PATCH 10/11] Rename volume --- bitnami/airflow/templates/deployment-scheduler.yaml | 8 ++++---- bitnami/airflow/templates/deployment-web.yaml | 8 ++++---- bitnami/airflow/templates/statefulset-worker.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index 67a95fafc..41d5cdd84 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -52,7 +52,7 @@ spec: imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: github-cloned-dag-files + - name: git-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -158,8 +158,8 @@ spec: mountPath: /opt/bitnami/airflow/dags/external {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files - mountPath: /opt/bitnami/airflow/dags/github + - name: git-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/git {{- end }} {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file @@ -180,7 +180,7 @@ spec: name: {{ .Values.airflow.dagsConfigMap }} {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files + - name: git-cloned-dag-files emptyDir: {} {{- end }} {{- if .Values.airflow.configurationConfigMap }} diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index 7a917f57b..d8659bf42 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -52,7 +52,7 @@ spec: imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: github-cloned-dag-files + - name: git-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -188,8 +188,8 @@ spec: mountPath: /opt/bitnami/airflow/dags/external {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files - mountPath: /opt/bitnami/airflow/dags/github + - name: git-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/git {{- end }} {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file @@ -210,7 +210,7 @@ spec: name: {{ .Values.airflow.dagsConfigMap }} {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files + - name: git-cloned-dag-files emptyDir: {} {{- end }} {{- if .Values.airflow.configurationConfigMap }} diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index 7caed12c0..223644791 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -56,7 +56,7 @@ spec: imagePullPolicy: {{ .Values.git.pullPolicy | quote }} command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] volumeMounts: - - name: github-cloned-dag-files + - name: git-cloned-dag-files mountPath: /dags {{- end }} containers: @@ -175,8 +175,8 @@ spec: mountPath: /opt/bitnami/airflow/dags/external {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files - mountPath: /opt/bitnami/airflow/dags/github + - name: git-cloned-dag-files + mountPath: /opt/bitnami/airflow/dags/git {{- end }} {{- if .Values.airflow.configurationConfigMap }} - name: custom-configuration-file @@ -195,7 +195,7 @@ spec: name: {{ .Values.airflow.dagsConfigMap }} {{- end }} {{- if .Values.airflow.cloneDagFilesFromGit.enabled }} - - name: github-cloned-dag-files + - name: git-cloned-dag-files emptyDir: {} {{- end }} {{- if .Values.airflow.configurationConfigMap }} From 1056190ffabdce3495531eccba19175a99f9cd15 Mon Sep 17 00:00:00 2001 From: tompizmor Date: Tue, 7 May 2019 17:35:15 +0200 Subject: [PATCH 11/11] Add airflow to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d8d394dd0..45cd165e2 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ $ helm search bitnami ## Bitnami charts - [Apache](https://github.com/bitnami/charts/tree/master/bitnami/apache) +- [Apache Airflow](https://github.com/bitnami/charts/tree/master/bitnami/airflow) - [Cassandra](https://github.com/bitnami/charts/tree/master/bitnami/cassandra) - [Elasticsearch](https://github.com/bitnami/charts/tree/master/bitnami/elasticsearch) - [ExternalDNS](https://github.com/bitnami/charts/tree/master/bitnami/external-dns)