[WIP] Improve Kubernetes System Tests Iteration Experience

chart/templates/_helpers.yaml

@@ -518,6 +518,28 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- printf "%s/logs" .Values.airflowHome }}
 {{- end }}
 
+{{- define "airflow_code_mount" -}}
+- name: code
+  mountPath: {{ printf "%s/airflow-core" .Values.airflowHome }}
+  subPath: airflow-core
+  readOnly: true
+- name: code
+  mountPath: {{ printf "%s/airflow-ctl" .Values.airflowHome }}
+  subPath: airflow-ctl
+  readOnly: true
+- name: code
+  mountPath: {{ printf "%s/task-sdk" .Values.airflowHome }}
+  subPath: task-sdk
+  readOnly: true
+{{- end }}
+{{- define "airflow_code_volume_claim" -}}
+  {{- if .Values.devMode.persistence.existingClaim }}
+    {{- .Values.devMode.persistence.existingClaim }}
+  {{- else }}
+    {{- printf "%s-code" .Release.Name }}
+  {{- end }}
+{{- end }}
+
 {{- define "airflow_logs_volume_claim" -}}
   {{- if .Values.logs.persistence.existingClaim }}
     {{- .Values.logs.persistence.existingClaim }}

chart/templates/api-server/api-server-deployment.yaml

@@ -173,6 +173,9 @@ spec:
             {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }}
               {{- include "airflow_api_server_config_mount" . | nindent 12 }}
             {{- end }}
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             {{- if .Values.logs.persistence.enabled }}
             - name: logs
               mountPath: {{ template "airflow_logs" . }}
@@ -235,6 +238,11 @@ spec:
         {{- end }}
         {{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
         {{- end }}
+        {{- if .Values.devMode.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
         {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/templates/dag-processor/dag-processor-deployment.yaml

@@ -172,6 +172,9 @@ spec:
             {{- if .Values.dagProcessor.extraVolumeMounts }}
               {{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
             {{- end }}
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             - name: logs
               mountPath: {{ template "airflow_logs" . }}
             {{- include "airflow_config_mount" . | nindent 12 }}
@@ -266,6 +269,11 @@ spec:
         {{- if .Values.dagProcessor.extraVolumes }}
           {{- tpl (toYaml .Values.dagProcessor.extraVolumes) . | nindent 8 }}
         {{- end }}
+        {{- if .Values.devMode.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
         {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/templates/dev-mode-code-pre-install-hook.yaml

@@ -0,0 +1,116 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+{{- if .Values.devMode.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-init-dev-code
+  labels:
+    tier: airflow
+    component: init-dev-code
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+  annotations:
+    helm.sh/hook: "pre-install"
+    helm.sh/hook-weight: "5"
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    # Increase hook timeout to avoid timing out in CI environments
+    helm.sh/hook-timeout-seconds: "300"
+spec:
+  template:
+    metadata:
+      labels:
+        tier: airflow
+        component: init-dev-code
+        release: {{ .Release.Name }}
+    spec:
+      securityContext: {{ include "airflowPodSecurityContext" (list . .Values) | nindent 8 }}
+      restartPolicy: OnFailure
+      containers:
+        - name: init-dev-code
+          image: {{ template "airflow_image" . }}
+          imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+          securityContext: {{ include "containerSecurityContext" (list . .Values) | nindent 12 }}
+          command:
+          - bash
+          - -c
+          - |
+            set -ex
+            # Give the PVC a moment to fully mount
+            echo "Waiting for PVC to be fully available..."
+            sleep 10
+
+            echo "Creating target directories if they don't exist..."
+            mkdir -p /mnt/target/airflow-core
+            mkdir -p /mnt/target/airflow-ctl
+            mkdir -p /mnt/target/task-sdk
+
+            echo "Checking if development code directories exist..."
+            # Skip copying if source directories don't exist or are empty
+            if [ -d "{{ printf "%s/airflow-core" .Values.airflowHome }}" ] && [ "$(ls -A {{ printf "%s/airflow-core" .Values.airflowHome }})" ]; then
+              echo "Copying airflow-core files..."
+              cp -rv {{ printf "%s/airflow-core" .Values.airflowHome }}/* /mnt/target/airflow-core/ || echo "Warning: Some files in airflow-core couldn't be copied"
+            else
+              echo "airflow-core directory doesn't exist or is empty, skipping..."
+              touch /mnt/target/airflow-core/.placeholder
+            fi
+
+            if [ -d "{{ printf "%s/airflow-ctl" .Values.airflowHome }}" ] && [ "$(ls -A {{ printf "%s/airflow-ctl" .Values.airflowHome }})" ]; then
+              echo "Copying airflow-ctl files..."
+              cp -rv {{ printf "%s/airflow-ctl" .Values.airflowHome }}/* /mnt/target/airflow-ctl/ || echo "Warning: Some files in airflow-ctl couldn't be copied"
+            else
+              echo "airflow-ctl directory doesn't exist or is empty, skipping..."
+              touch /mnt/target/airflow-ctl/.placeholder
+            fi
+
+            if [ -d "{{ printf "%s/task-sdk" .Values.airflowHome }}" ] && [ "$(ls -A {{ printf "%s/task-sdk" .Values.airflowHome }})" ]; then
+              echo "Copying task-sdk files..."
+              cp -rv {{ printf "%s/task-sdk" .Values.airflowHome }}/* /mnt/target/task-sdk/ || echo "Warning: Some files in task-sdk couldn't be copied"
+            else
+              echo "task-sdk directory doesn't exist or is empty, skipping..."
+              touch /mnt/target/task-sdk/.placeholder
+            fi
+
+            # List directories to verify they exist
+            echo "Listing target directories content:"
+            ls -la /mnt/target/airflow-core
+            ls -la /mnt/target/airflow-ctl
+            ls -la /mnt/target/task-sdk
+
+            # Add a marker file to indicate this job has run
+            touch {{ printf "%s/airflow-core/.init-complete" .Values.airflowHome }}
+
+            echo "All required development code directories verified or created."
+          volumeMounts:
+            - name: code
+              mountPath: /mnt/target/airflow-core
+              subPath: airflow-core
+            - name: code
+              mountPath: /mnt/target/airflow-ctl
+              subPath: airflow-ctl
+            - name: code
+              mountPath: /mnt/target/task-sdk
+              subPath: task-sdk
+      volumes:
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+{{- end }}

chart/templates/dev-mode-code-volume-claim.yaml

@@ -0,0 +1,55 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+######################################
+## Airflow Source Code PersistentVolumeClaim, used for mounting source code with `dev` mode
+######################################
+{{- if .Values.devMode.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ template "airflow_code_volume_claim" . }}
+  labels:
+    tier: airflow
+    component: code-pvc
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    helm.sh/hook: "pre-install"
+    helm.sh/hook-weight: "-10"
+    {{- with .Values.devMode.persistence.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: {{ .Values.devMode.persistence.size | quote }}
+  {{- if .Values.devMode.persistence.storageClassName }}
+  {{- if (eq "-" .Values.devMode.persistence.storageClassName) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: {{ tpl .Values.devMode.persistence.storageClassName . | quote }}
+  {{- end }}
+  {{- end }}
+{{- end }}

chart/templates/scheduler/scheduler-deployment.yaml

@@ -236,6 +236,9 @@ spec:
               subPath: pod_template_file.yaml
               readOnly: true
             {{- end }}
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             - name: logs
               mountPath: {{ template "airflow_logs" . }}
             {{- include "airflow_config_mount" . | nindent 12 }}
@@ -327,6 +330,11 @@ spec:
         {{- if .Values.scheduler.extraVolumes }}
           {{- tpl (toYaml .Values.scheduler.extraVolumes) . | nindent 8 }}
         {{- end }}
+        {{- if .Values.devMode.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
   {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/templates/triggerer/triggerer-deployment.yaml

@@ -255,6 +255,9 @@ spec:
               {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.env) $ | nindent 12 }}
           {{- end }}
           volumeMounts:
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             - name: logs
               mountPath: {{ template "airflow_logs" . }}
             {{- if .Values.volumeMounts }}
@@ -296,6 +299,11 @@ spec:
         {{- if .Values.triggerer.extraVolumes }}
           {{- tpl (toYaml .Values.triggerer.extraVolumes) . | nindent 8 }}
         {{- end }}
+        {{- if .Values.devMode.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
   {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/templates/webserver/webserver-deployment.yaml

@@ -150,6 +150,9 @@ spec:
             {{- if .Values.webserver.extraVolumeMounts }}
               {{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
             {{- end }}
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             {{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
               {{- include "airflow_webserver_config_mount" . | nindent 12 }}
             {{- end }}
@@ -285,6 +288,11 @@ spec:
         {{- end }}
         {{- end }}
         {{- end }}
+        {{- if .Values.devMode.persistence.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
         {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/templates/workers/worker-deployment.yaml

@@ -167,6 +167,9 @@ spec:
             - name: logs
               mountPath: {{ template "airflow_logs" . }}
             {{- include "airflow_config_mount" . | nindent 12 }}
+            {{- if .Values.devMode.enabled }}
+              {{- include "airflow_code_mount" . | nindent 12 }}
+            {{- end }}
             - name: config
               mountPath: {{ .Values.kerberos.configPath | quote }}
               subPath: krb5.conf
@@ -432,6 +435,11 @@ spec:
           {{- include "git_sync_ssh_key_volume" . | indent 8 }}
         {{- end }}
         {{- end }}
+        {{- if .Values.devMode.enabled }}
+        - name: code
+          persistentVolumeClaim:
+            claimName: {{ template "airflow_code_volume_claim" . }}
+        {{- end }}
   {{- if .Values.logs.persistence.enabled }}
         - name: logs
           persistentVolumeClaim:

chart/values.schema.json

@@ -39,6 +39,54 @@
             "default": null,
             "x-docsSection": null
         },
+        "devMode": {
+            "description": "Development mode. If true, the chart will mount the source code into the deployments.",
+            "type": "object",
+            "x-docsSection": "Airflow",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "description": "Whether to enable development mode.",
+                    "type": "boolean",
+                    "default": false
+                },
+                "persistence": {
+                    "type": "object",
+                    "description": "Persistence configuration for development mode.",
+                    "properties": {
+                        "size": {
+                            "description": "Volume size for the source code",
+                            "type": "string",
+                            "default": "128Mi"
+                        },
+                        "annotations": {
+                            "description": "Annotations for the source code PVC",
+                            "type": "object",
+                            "default": {},
+                            "additionalProperties": {
+                                "type": "string"
+                            }
+                        },
+                        "storageClassName": {
+                            "description": "If using a custom StorageClass, pass name here (templated).",
+                            "type": [
+                                "string",
+                                "null"
+                            ],
+                            "default": null
+                        },
+                        "existingClaim": {
+                            "description": "The name of an existing PVC to use.",
+                            "type": [
+                                "string",
+                                "null"
+                            ],
+                            "default": null
+                        }
+                    }
+                }
+            }
+        },
         "nameOverride": {
             "description": "Override the name of the chart",
             "type": "string",