feat(airflow): upgrade airflow to 2.6.0 (#677)

* update chart + add support for plural redis app

Signed-off-by: David van der Spek <[email protected]>

* fix oidc for newer auth lib

Signed-off-by: David van der Spek <[email protected]>

* bump app version

* bump image version

* ignore vscode workspace files

* fix redis default, empty string, overwrite by tpl

* cleanup old airflow image building CI

Signed-off-by: David van der Spek <[email protected]>

* bump chart

Signed-off-by: David van der Spek <[email protected]>

---------

Signed-off-by: David van der Spek <[email protected]>
Co-authored-by: Hans Rauer <[email protected]>

Author: davidspek

.github/workflows/docker.yml

@@ -56,32 +56,6 @@ jobs:
         docker build -f sentry/Dockerfile -t gcr.io/pluralsh/getsentry/sentry:21.5.1-extended -t dkr.plural.sh/sentry/sentry:21.5.1-extended .
         docker push gcr.io/pluralsh/getsentry/sentry:21.5.1-extended
         docker push dkr.plural.sh/sentry/sentry:21.5.1-extended
-  docker-airflow:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    steps:
-    - uses: actions/checkout@v3
-    - uses: google-github-actions/auth@v1
-      with:
-        workload_identity_provider: 'projects/${{ secrets.GOOGLE_PROJECT_ID }}/locations/global/workloadIdentityPools/github/providers/github'
-        service_account: '[email protected]'
-        token_format: 'access_token'
-        create_credentials_file: true
-    - uses: google-github-actions/[email protected]
-    - name: Login to gcr
-      run: gcloud auth configure-docker -q
-    - name: Login to plural registry
-      uses: docker/login-action@v2
-      with:
-        registry: dkr.plural.sh
-        username: [email protected]
-        password: ${{ secrets.PLURAL_ACCESS_TOKEN }}
-    - run: |
-        docker build -t gcr.io/pluralsh/apache/airflow:2.2.5-python3.8-dbt -t dkr.plural.sh/airflow/apache/airflow:2.2.5-python3.8-dbt airflow
-        docker push gcr.io/pluralsh/apache/airflow:2.2.5-python3.8-dbt
-        docker push dkr.plural.sh/airflow/apache/airflow:2.2.5-python3.8-dbt
   docker-superset:
     runs-on: ubuntu-latest
     permissions:

.gitignore

@@ -2,3 +2,4 @@
 .*.swp
 .DS_Store
 /.idea/
+*.code-workspace

airflow/Dockerfile

@@ -4,12 +4,12 @@ dependencies:
   version: 0.1.1
 - name: airflow
   repository: https://airflow-helm.github.io/charts
-  version: 8.5.2
+  version: 8.6.1
 - name: postgres
   repository: https://pluralsh.github.io/module-library
   version: 0.1.16
 - name: test-base
   repository: https://pluralsh.github.io/module-library
   version: 0.1.3
-digest: sha256:1aafdd32c53f4986f320f15fb2f6c31673100b62e0db1a9ec42deef2c3010b29
-generated: "2022-12-30T11:24:13.427115-05:00"
+digest: sha256:6289d3b707a41a305fa96650ea599ed9be1f889175d250d648631cb568bced8b
+generated: "2023-03-23T14:51:25.535042+01:00"

airflow/helm/airflow/Chart.lock

@@ -2,8 +2,8 @@ apiVersion: v2
 name: airflow
 description: A Helm chart for airflow deployable on plural
 type: application
-version: 0.3.33
-appVersion: "1.16.0"
+version: 0.3.34
+appVersion: "2.6.0"
 sources:
   - https://github.com/pluralsh/plural-artifacts/airflow/helm/airflow
 maintainers:
@@ -14,7 +14,7 @@ dependencies:
   version: 0.1.1
   repository: https://pluralsh.github.io/module-library
 - name: airflow
-  version: 8.5.2
+  version: 8.6.1
   repository: https://airflow-helm.github.io/charts
 - name: postgres
   version: 0.1.16

airflow/helm/airflow/Chart.yaml

@@ -17,6 +17,11 @@ spec:
     name: postgres
     repo: postgres
     version: '>= 0.1.6'
+  - type: helm
+    name: redis
+    repo: redis
+    version: '>= 0.1.6'
+    optional: true
   - type: terraform
     name: aws
     repo: airflow
@@ -31,4 +36,4 @@ spec:
     name: gcp
     repo: airflow
     version: '>= 0.1.1'
-    optional: true
+    optional: true

airflow/helm/airflow/charts/airflow-8.5.2.tgz

@@ -1,9 +1,9 @@
-{{- if .Values.airflow.redis.enabled }}
+{{- if or .Values.airflow.redis.enabled .Values.secrets.createPluralRedisSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
   name: airflow-redis-password
-stringData:
+data:
   redis-password: {{ .Values.secrets.redis_password | b64enc | quote }}
 {{- end }}
 {{- if .Values.sshConfig.enabled }}

airflow/helm/airflow/charts/airflow-8.6.1.tgz

@@ -12,6 +12,7 @@ httpConfig:
   password: dummy
 
 secrets:
+  createPluralRedisSecret: false
   redis_password: example
 
 test-base:
@@ -298,12 +299,19 @@ airflow:
 
       persistence:
         enabled: false
+  
+  externalRedis:
+    host: ""
+    port: 6379
+    databaseNumber: 1
+    passwordSecret: airflow-redis-password
+    passwordSecretKey: redis-password
 
   airflow:
     executor: CeleryExecutor
     image:
-      repository: dkr.plural.sh/airflow/apache/airflow
-      tag: 2.1.4-python3.8-dbt-2
+      repository: ghcr.io/pluralsh/containers/apache/airflow
+      tag: 2.6.0-python3.10-plural1.4.0
 
     config:
       ## security
@@ -387,7 +395,7 @@ airflow:
     podDisruptionBudget:
       enabled: true
 
-      maxUnavailable: "20%"
+      maxUnavailable: "50%"
 
     autoscaling:
       enabled: true