From 492c2c386600875498da2900605cc074ebdf0564 Mon Sep 17 00:00:00 2001
From: juan131 <juan@bitnami.com>
Date: Fri, 24 May 2019 17:07:53 +0200
Subject: [PATCH 1/3] [bitnami/airflow] Add sidecar container to update git
 repository periodically

Signed-off-by: juan131 <juan@bitnami.com>
---
 bitnami/airflow/Chart.yaml                    |  2 +-
 bitnami/airflow/README.md                     | 11 ++++---
 bitnami/airflow/templates/NOTES.txt           |  4 ++-
 bitnami/airflow/templates/_helpers.tpl        | 31 +++++++++++++++++--
 .../templates/deployment-scheduler.yaml       | 26 ++++++++++++++--
 bitnami/airflow/templates/deployment-web.yaml | 26 ++++++++++++++--
 .../airflow/templates/statefulset-worker.yaml | 26 ++++++++++++++--
 bitnami/airflow/values-production.yaml        |  7 +++--
 bitnami/airflow/values.yaml                   |  5 +--
 9 files changed, 115 insertions(+), 23 deletions(-)

diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml
index 5b179a92a..3ad94834e 100644
--- a/bitnami/airflow/Chart.yaml
+++ b/bitnami/airflow/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: airflow
-version: 0.0.4
+version: 0.1.0
 appVersion: 1.10.3
 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows.
 keywords:
diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md
index 1f6b443f4..a48dd5f96 100644
--- a/bitnami/airflow/README.md
+++ b/bitnami/airflow/README.md
@@ -79,7 +79,8 @@ The following tables lists the configurable parameters of the Kafka chart and th
 | `airflow.loadExamples`                    | Switch to load some Airflow examples                                                        | `true`                                                       |
 | `airflow.cloneDagFilesFromGit.enabled`    | Enable in order to download DAG files from git repository.                                  | `false`                                                      |
 | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from                                                    | `nil`                                                        |
-| `airflow.cloneDagFilesFromGit.revision`   | Revision from repository to checkout                                                        | `nil`                                                        |
+| `airflow.cloneDagFilesFromGit.branch`     | Branch from repository to checkout                                                          | `nil`                                                        |
+| `airflow.cloneDagFilesFromGit.interval`   | Interval to pull the repository on sidecar container                                        | `nil`                                                        |
 | `airflow.baseUrl`                         | URL used to access to airflow web ui                                                        | `nil`                                                        |
 | `airflow.worker.port`                     | Airflow Worker port                                                                         | `8793`                                                       |
 | `airflow.worker.replicas`                 | Number of Airflow Worker replicas                                                           | `2`                                                          |
@@ -188,12 +189,12 @@ You can manually create a config map containing all your DAG files and then pass
 
 ### Option 3: Get your DAG files from a git repository
 
-You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. In order to do that, you can deploy airflow with the following options:
+You can store all your DAG files on a GitHub repository and then clone to the Airflow pods with an initContainer. The repository will be periodically updated using a sidecar container. In order to do that, you can deploy airflow with the following options:
 
 ```console
 helm install --name my-release bitnami/airflow \
              --set airflow.cloneDagFilesFromGit.enabled=true \
              --set airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY \
-             --set airflow.cloneDagFilesFromGit.revision=master
-
-```
\ No newline at end of file
+             --set airflow.cloneDagFilesFromGit.branch=master
+             --set airflow.cloneDagFilesFromGit.interval=3600
+```
diff --git a/bitnami/airflow/templates/NOTES.txt b/bitnami/airflow/templates/NOTES.txt
index 67023e306..92ddb4555 100644
--- a/bitnami/airflow/templates/NOTES.txt
+++ b/bitnami/airflow/templates/NOTES.txt
@@ -75,4 +75,6 @@ host. To configure Airflow with the URL of your service:
   echo User:    {{ .Values.airflow.auth.username }}
   echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath="{.data.airflow-password}" | base64 --decode)
 
-  {{- end  }}
\ No newline at end of file
+{{- end  }}
+
+{{ include "airflow.validateValues" . }}
diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl
index a7aeff416..5e4ccc066 100644
--- a/bitnami/airflow/templates/_helpers.tpl
+++ b/bitnami/airflow/templates/_helpers.tpl
@@ -146,7 +146,6 @@ Also, we can't use a single if because lazy evaluation is not an option
 {{- end -}}
 {{- end -}}
 
-
 {{/*
 Return the proper Docker Image Registry Secret Names
 */}}
@@ -233,4 +232,32 @@ Get the secret name
 {{- else -}}
 {{- printf "%s" (include "airflow.fullname" .) -}}
 {{- end -}}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
+
+{{/*
+Compile all warnings into a single message, and call fail.
+*/}}
+{{- define "airflow.validateValues" -}}
+{{- $messages := list -}}
+{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit" .) -}}
+{{- $messages := without $messages "" -}}
+{{- $message := join "\n" $messages -}}
+
+{{- if $message -}}
+{{-   printf "\nVALUES VALIDATION:\n%s" $message | fail -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.repository" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.cloneDagFilesFromGit" -}}
+{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.repository) -}}
+airflow: airflow.cloneDagFilesFromGit.repository
+    The repository must be provided when enabling downloading DAG files
+    from git repository (--set airflow.cloneDagFilesFromGit.repository="xxx")
+{{- end -}}
+{{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.branch) -}}
+airflow: airflow.cloneDagFilesFromGit.branch
+    The branch must be provided when enabling downloading DAG files
+    from git repository (--set airflow.cloneDagFilesFromGit.branch="xxx")
+{{- end -}}
+{{- end -}}
diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml
index 41d5cdd84..c24ccc30b 100644
--- a/bitnami/airflow/templates/deployment-scheduler.yaml
+++ b/bitnami/airflow/templates/deployment-scheduler.yaml
@@ -50,12 +50,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-scheduler
         image: {{ template "airflow.schedulerImage" . }}
         imagePullPolicy: {{ .Values.schedulerImage.pullPolicy | quote }}
@@ -187,4 +207,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
\ No newline at end of file
+      {{- end }}
diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml
index d8659bf42..4bf8ce7bd 100644
--- a/bitnami/airflow/templates/deployment-web.yaml
+++ b/bitnami/airflow/templates/deployment-web.yaml
@@ -50,12 +50,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-web
         image: {{ template "airflow.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
@@ -217,4 +237,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
\ No newline at end of file
+      {{- end }}
diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml
index 223644791..f8f875ad0 100644
--- a/bitnami/airflow/templates/statefulset-worker.yaml
+++ b/bitnami/airflow/templates/statefulset-worker.yaml
@@ -54,12 +54,32 @@ spec:
       - name: git-clone-repository
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
-        command: [ '/bin/sh', '-c' , 'git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} /dags && cd /dags && git checkout {{ .Values.airflow.cloneDagFilesFromGit.revision }}']
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
         volumeMounts:
         - name: git-cloned-dag-files
           mountPath: /dags
-      {{- end }}
       containers:
+      - name: clone-repo-sidecar
+        image: "{{ template "git.image" . }}"
+        imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
+        command:
+        - /bin/bash
+        - -ec
+        - |
+          while true; do
+              cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
+              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+          done
+        volumeMounts:
+        - name: git-cloned-dag-files
+          mountPath: /dags
+      {{- else }}
+      containers:
+      {{- end }}
       - name: airflow-worker
         image: "{{ template "airflow.workerImage" . }}"
         imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}"
@@ -202,4 +222,4 @@ spec:
       - name: custom-configuration-file
         configMap:
           name: {{ .Values.airflow.configurationConfigMap }}
-      {{- end }}
\ No newline at end of file
+      {{- end }}
diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml
index d3ca6d1b0..0affa40fa 100644
--- a/bitnami/airflow/values-production.yaml
+++ b/bitnami/airflow/values-production.yaml
@@ -120,8 +120,9 @@ airflow:
   ##
   cloneDagFilesFromGit:
     enabled: false
-    repository:
-    revision:
+    # repository:
+    # branch:
+    # interval:
   ## URL used to access to airflow web ui
   ##
   baseUrl: http://airflow.local
@@ -354,4 +355,4 @@ metrics:
 
   ## Metrics exporter pod Annotation and Labels
   # podAnnotations: {}
-  # podLabels: {}
\ No newline at end of file
+  # podLabels: {}
diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml
index 286a55906..d430b4301 100644
--- a/bitnami/airflow/values.yaml
+++ b/bitnami/airflow/values.yaml
@@ -120,8 +120,9 @@ airflow:
   ##
   cloneDagFilesFromGit:
     enabled: true
-    repository: https://github.com/tompizmor/sample-airflow-dag
-    revision: master
+    # repository:
+    # branch:
+    # interval:
   ## URL used to access to airflow web ui
   ##
   # baseUrl:

From 305dda20292afd5e1ddc2e5a08f195fa5ddeb3ad Mon Sep 17 00:00:00 2001
From: juan131 <juan@bitnami.com>
Date: Fri, 24 May 2019 17:25:03 +0200
Subject: [PATCH 2/3] Use 60 seconds as default interval

Signed-off-by: juan131 <juan@bitnami.com>
---
 bitnami/airflow/README.md                           | 2 +-
 bitnami/airflow/templates/_helpers.tpl              | 8 ++++++--
 bitnami/airflow/templates/deployment-scheduler.yaml | 2 +-
 bitnami/airflow/templates/deployment-web.yaml       | 2 +-
 bitnami/airflow/templates/statefulset-worker.yaml   | 2 +-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md
index a48dd5f96..95c6720fb 100644
--- a/bitnami/airflow/README.md
+++ b/bitnami/airflow/README.md
@@ -196,5 +196,5 @@ helm install --name my-release bitnami/airflow \
              --set airflow.cloneDagFilesFromGit.enabled=true \
              --set airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY \
              --set airflow.cloneDagFilesFromGit.branch=master
-             --set airflow.cloneDagFilesFromGit.interval=3600
+             --set airflow.cloneDagFilesFromGit.interval=60
 ```
diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl
index 5e4ccc066..034dd27c6 100644
--- a/bitnami/airflow/templates/_helpers.tpl
+++ b/bitnami/airflow/templates/_helpers.tpl
@@ -239,7 +239,8 @@ Compile all warnings into a single message, and call fail.
 */}}
 {{- define "airflow.validateValues" -}}
 {{- $messages := list -}}
-{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit" .) -}}
+{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit.repository" .) -}}
+{{- $messages := append $messages (include "airflow.validateValues.cloneDagFilesFromGit.branch" .) -}}
 {{- $messages := without $messages "" -}}
 {{- $message := join "\n" $messages -}}
 
@@ -249,12 +250,15 @@ Compile all warnings into a single message, and call fail.
 {{- end -}}
 
 {{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.repository" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}}
-{{- define "airflow.validateValues.cloneDagFilesFromGit" -}}
+{{- define "airflow.validateValues.cloneDagFilesFromGit.repository" -}}
 {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.repository) -}}
 airflow: airflow.cloneDagFilesFromGit.repository
     The repository must be provided when enabling downloading DAG files
     from git repository (--set airflow.cloneDagFilesFromGit.repository="xxx")
 {{- end -}}
+{{- end -}}
+{{/* Validate values of Airflow - "airflow.cloneDagFilesFromGit.branch" must be provided when "airflow.cloneDagFilesFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.cloneDagFilesFromGit.branch" -}}
 {{- if and .Values.airflow.cloneDagFilesFromGit.enabled (empty .Values.airflow.cloneDagFilesFromGit.branch) -}}
 airflow: airflow.cloneDagFilesFromGit.branch
     The branch must be provided when enabling downloading DAG files
diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml
index c24ccc30b..cea2d1094 100644
--- a/bitnami/airflow/templates/deployment-scheduler.yaml
+++ b/bitnami/airflow/templates/deployment-scheduler.yaml
@@ -68,7 +68,7 @@ spec:
         - |
           while true; do
               cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+              sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
           done
         volumeMounts:
         - name: git-cloned-dag-files
diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml
index 4bf8ce7bd..71b17a81f 100644
--- a/bitnami/airflow/templates/deployment-web.yaml
+++ b/bitnami/airflow/templates/deployment-web.yaml
@@ -68,7 +68,7 @@ spec:
         - |
           while true; do
               cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+              sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
           done
         volumeMounts:
         - name: git-cloned-dag-files
diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml
index f8f875ad0..3f6c2d3ed 100644
--- a/bitnami/airflow/templates/statefulset-worker.yaml
+++ b/bitnami/airflow/templates/statefulset-worker.yaml
@@ -72,7 +72,7 @@ spec:
         - |
           while true; do
               cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-              sleep {{ default "3600" .Values.airflow.cloneDagFilesFromGit.interval }}
+              sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
           done
         volumeMounts:
         - name: git-cloned-dag-files

From 60c90851d887a5af14c67757a6f2fac7a823075f Mon Sep 17 00:00:00 2001
From: juan131 <juan@bitnami.com>
Date: Fri, 24 May 2019 17:26:27 +0200
Subject: [PATCH 3/3] Rename sidecar container to 'git-repo-syncer'

Signed-off-by: juan131 <juan@bitnami.com>
---
 bitnami/airflow/templates/deployment-scheduler.yaml | 2 +-
 bitnami/airflow/templates/deployment-web.yaml       | 2 +-
 bitnami/airflow/templates/statefulset-worker.yaml   | 2 +-
 bitnami/airflow/values.yaml                         | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml
index cea2d1094..bd07634e4 100644
--- a/bitnami/airflow/templates/deployment-scheduler.yaml
+++ b/bitnami/airflow/templates/deployment-scheduler.yaml
@@ -59,7 +59,7 @@ spec:
         - name: git-cloned-dag-files
           mountPath: /dags
       containers:
-      - name: clone-repo-sidecar
+      - name: git-repo-syncer
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
         command:
diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml
index 71b17a81f..536016e6d 100644
--- a/bitnami/airflow/templates/deployment-web.yaml
+++ b/bitnami/airflow/templates/deployment-web.yaml
@@ -59,7 +59,7 @@ spec:
         - name: git-cloned-dag-files
           mountPath: /dags
       containers:
-      - name: clone-repo-sidecar
+      - name: git-repo-syncer
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
         command:
diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml
index 3f6c2d3ed..0960b8035 100644
--- a/bitnami/airflow/templates/statefulset-worker.yaml
+++ b/bitnami/airflow/templates/statefulset-worker.yaml
@@ -63,7 +63,7 @@ spec:
         - name: git-cloned-dag-files
           mountPath: /dags
       containers:
-      - name: clone-repo-sidecar
+      - name: git-repo-syncer
         image: "{{ template "git.image" . }}"
         imagePullPolicy: {{ .Values.git.pullPolicy | quote }}
         command:
diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml
index d430b4301..3e0ef87cf 100644
--- a/bitnami/airflow/values.yaml
+++ b/bitnami/airflow/values.yaml
@@ -119,7 +119,7 @@ airflow:
   ## Enable in order to download DAG files from git repository.
   ##
   cloneDagFilesFromGit:
-    enabled: true
+    enabled: false
     # repository:
     # branch:
     # interval: