diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml index 5b179a92a..3ad94834e 100644 --- a/bitnami/airflow/Chart.yaml +++ b/bitnami/airflow/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: airflow -version: 0.0.4 +version: 0.1.0 appVersion: 1.10.3 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows. keywords: diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 1f6b443f4..95c6720fb 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -79,7 +79,8 @@ The following tables lists the configurable parameters of the Kafka chart and th | `airflow.loadExamples` | Switch to load some Airflow examples | `true` | | `airflow.cloneDagFilesFromGit.enabled` | Enable in order to download DAG files from git repository. | `false` | | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from | `nil` | -| `airflow.cloneDagFilesFromGit.revision` | Revision from repository to checkout | `nil` | +| `airflow.cloneDagFilesFromGit.branch` | Branch from repository to checkout | `nil` | +| `airflow.cloneDagFilesFromGit.interval` | Interval to pull the repository on sidecar container | `nil` | | `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | @@ -188,12 +189,12 @@ You can manually create a config map containing all your DAG files and then pass ### Option 3: Get your DAG files from a git repository -You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. In order to do that, you can deploy airflow with the following options: +You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. The repository will be periodically updated using a sidecar container. In order to do that, you can deploy airflow with the following options: ```console helm install --name my-release bitnami/airflow \ --set airflow.cloneDagFilesFromGit.enabled=true \ --set airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY \ - --set airflow.cloneDagFilesFromGit.revision=master - -``` \ No newline at end of file + --set airflow.cloneDagFilesFromGit.branch=master + --set airflow.cloneDagFilesFromGit.interval=60 +``` diff --git a/bitnami/airflow/templates/NOTES.txt b/bitnami/airflow/templates/NOTES.txt index 67023e306..92ddb4555 100644 --- a/bitnami/airflow/templates/NOTES.txt +++ b/bitnami/airflow/templates/NOTES.txt @@ -75,4 +75,6 @@ host. To configure Airflow with the URL of your service: echo User: {{ .Values.airflow.auth.username }} echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode) - {{- end }} \ No newline at end of file +{{- end }} + +{{ include "airflow.validateValues" . }} diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index a7aeff416..034dd27c6 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -146,7 +146,6 @@ Also, we can't use a single if because lazy evaluation is not an option {{- end -}} {{- end -}} - {{/* Return the proper Docker Image Registry Secret Names */}} @@ -233,4 +232,36 @@ Get the secret name {{- else -}} {{- printf "%s" (include "airflow.fullname" .) -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "airflow.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit.repository" .) -}} +{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit.branch" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.repository" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}} +{{- define "airflow.validateValues.cloneDagFilesFromGit.repository" -}} +{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.repository) -}} +airflow: airflow.cloneDagFilesFromGit.repository + The repository must be provided when enabling downloading DAG files + from git repository (--set airflow.cloneDagFilesFromGit.repository="xxx") +{{- end -}} +{{- end -}} +{{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.branch" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}} +{{- define "airflow.validateValues.cloneDagFilesFromGit.branch" -}} +{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.branch) -}} +airflow: airflow.cloneDagFilesFromGit.branch + The branch must be provided when enabling downloading DAG files + from git repository (--set airflow.cloneDagFilesFromGit.branch="xxx") +{{- end -}} +{{- end -}} diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index 41d5cdd84..bd07634e4 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -50,12 +50,32 @@ spec: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} - command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + command: + - /bin/bash + - -ec + - | + git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags volumeMounts: - name: git-cloned-dag-files mountPath: /dags - {{- end }} containers: + - name: git-repo-syncer + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + while true; do + cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }} + sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }} + done + volumeMounts: + - name: git-cloned-dag-files + mountPath: /dags + {{- else }} + containers: + {{- end }} - name: airflow-scheduler image: {{ template "airflow.schedulerImage" . }} imagePullPolicy: {{ .Values.schedulerImage.pullPolicy | quote }} @@ -187,4 +207,4 @@ spec: - name: custom-configuration-file configMap: name: {{ .Values.airflow.configurationConfigMap }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index d8659bf42..536016e6d 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -50,12 +50,32 @@ spec: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} - command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + command: + - /bin/bash + - -ec + - | + git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags volumeMounts: - name: git-cloned-dag-files mountPath: /dags - {{- end }} containers: + - name: git-repo-syncer + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + while true; do + cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }} + sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }} + done + volumeMounts: + - name: git-cloned-dag-files + mountPath: /dags + {{- else }} + containers: + {{- end }} - name: airflow-web image: {{ template "airflow.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} @@ -217,4 +237,4 @@ spec: - name: custom-configuration-file configMap: name: {{ .Values.airflow.configurationConfigMap }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index 223644791..0960b8035 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -54,12 +54,32 @@ spec: - name: git-clone-repository image: "{{ template "git.image" . }}" imagePullPolicy: {{ .Values.git.pullPolicy | quote }} - command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}'] + command: + - /bin/bash + - -ec + - | + git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags volumeMounts: - name: git-cloned-dag-files mountPath: /dags - {{- end }} containers: + - name: git-repo-syncer + image: "{{ template "git.image" . }}" + imagePullPolicy: {{ .Values.git.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + while true; do + cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }} + sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }} + done + volumeMounts: + - name: git-cloned-dag-files + mountPath: /dags + {{- else }} + containers: + {{- end }} - name: airflow-worker image: "{{ template "airflow.workerImage" . }}" imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}" @@ -202,4 +222,4 @@ spec: - name: custom-configuration-file configMap: name: {{ .Values.airflow.configurationConfigMap }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index d3ca6d1b0..0affa40fa 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -120,8 +120,9 @@ airflow: ## cloneDagFilesFromGit: enabled: false - repository: - revision: + # repository: + # branch: + # interval: ## URL used to access to airflow web ui ## baseUrl: http://airflow.local @@ -354,4 +355,4 @@ metrics: ## Metrics exporter pod Annotation and Labels # podAnnotations: {} - # podLabels: {} \ No newline at end of file + # podLabels: {} diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 286a55906..3e0ef87cf 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -119,9 +119,10 @@ airflow: ## Enable in order to download DAG files from git repository. ## cloneDagFilesFromGit: - enabled: true - repository: https://github.com/tompizmor/sample-airflow-dag - revision: master + enabled: false + # repository: + # branch: + # interval: ## URL used to access to airflow web ui ## # baseUrl: