feat: Airflow 3.0.1 (experimental) (#630)

* wip: working tests (except auth: opa, oidc)

* wip: changing python test files

* single health.py for all airflow versions

* single metrics.py for all airflow versions

* update tests with new commons scripts

* tests: use "airflow-latest" instead of "airflow"

* cleanup and code comments

* restore deleted env

* use correct webserver service

* restore operator to release list

* test: fix oidc

* wip: get logging tests to work post-merge

* make env-vars version-specific

* fixed resolution of webserver url for execution api

* relaxed default resources

* update test defs for oidc/opa

* changelog

* Update tests/templates/kuttl/logging/51-assert.yaml.j2

Co-authored-by: Malte Sander <[email protected]>

* code review changes

* code review changes

* move containerdebug cmd to a function

* replaced random key with hard-coded one with comment

* cleanup and better comment

* for 3.x: only scheduler updates FAB permissions and restrict workers

* make api-server env-vars role dependent

---------

Co-authored-by: Razvan-Daniel Mihai <[email protected]>
Co-authored-by: Malte Sander <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -10,6 +10,7 @@
   - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation.
   - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`.
 - Add support for airflow `2.10.5` ([#625]).
+- Add experimental support for airflow `3.0.1` ([#630]).
 
 ### Changed
 
@@ -41,6 +42,7 @@
 [#623]: https://github.com/stackabletech/airflow-operator/pull/623
 [#624]: https://github.com/stackabletech/airflow-operator/pull/624
 [#625]: https://github.com/stackabletech/airflow-operator/pull/625
+[#630]: https://github.com/stackabletech/airflow-operator/pull/630
 
 ## [25.3.0] - 2025-03-21
 

docs/modules/airflow/pages/usage-guide/storage-resources.adoc

@@ -5,9 +5,9 @@ include::home:concepts:stackable_resource_requests.adoc[]
 
 A minimal HA setup consisting of 2 schedulers, 2 workers and 2 webservers has the following https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[resource requirements]:
 
-* `5700m` CPU request
+* `8700m` CPU request
 * `17400m` CPU limit
-* `10752Mi` memory request and limit
+* `15872Mi` memory request and limit
 
 Corresponding to the values above, the operator uses the following resource defaults:
 
@@ -18,24 +18,24 @@ spec:
     config:
       resources:
         cpu:
-          min: 500m
+          min: "1"
           max: "2"
         memory:
-          limit: 512Mi
+          limit: 1Gi
   celeryExecutors:
     config:
       resources:
         cpu:
-          min: 500m
+          min: "1"
           max: "2"
         memory:
-          limit: 2Gi
+          limit: 3Gi
   webservers:
     config:
       resources:
         cpu:
-          min: 500m
+          min: "1"
           max: "2"
         memory:
-          limit: 2Gi
+          limit: 3Gi
 ----

docs/modules/airflow/partials/supported-versions.adoc

@@ -2,6 +2,7 @@
 // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching
 // Stackable Platform documentation.
 
+- 3.0.1 (experimental)
 - 2.10.5
 - 2.10.4 (deprecated)
 - 2.9.3 (LTS)

rust/operator-binary/src/airflow_controller.rs

@@ -957,7 +957,8 @@ fn build_server_rolegroup_statefulset(
         .context(GracefulShutdownSnafu)?;
 
     let mut airflow_container_args = Vec::new();
-    airflow_container_args.extend(airflow_role.get_commands(authentication_config));
+    airflow_container_args
+        .extend(airflow_role.get_commands(authentication_config, resolved_product_image));
 
     airflow_container
         .image_from_product_image(resolved_product_image)
@@ -980,6 +981,7 @@ fn build_server_rolegroup_statefulset(
             authentication_config,
             authorization_config,
             git_sync_resources,
+            resolved_product_image,
         )
         .context(BuildStatefulsetEnvVarsSnafu)?,
     );
@@ -1170,7 +1172,10 @@ fn build_server_rolegroup_statefulset(
             match_labels: Some(statefulset_match_labels.into()),
             ..LabelSelector::default()
         },
-        service_name: Some(rolegroup_ref.object_name()),
+        service_name: Some(format!(
+            "{name}-metrics",
+            name = rolegroup_ref.object_name()
+        )),
         template: pod_template,
         volume_claim_templates: pvcs,
         ..StatefulSetSpec::default()
@@ -1263,6 +1268,7 @@ fn build_executor_template_config_map(
             env_overrides,
             merged_executor_config,
             git_sync_resources,
+            resolved_product_image,
         ))
         .add_volume_mounts(airflow.volume_mounts())
         .context(AddVolumeMountSnafu)?

rust/operator-binary/src/crd/mod.rs

@@ -9,7 +9,7 @@ use stackable_operator::{
         cache::UserInformationCache,
         cluster_operation::ClusterOperation,
         opa::OpaConfig,
-        product_image_selection::ProductImage,
+        product_image_selection::{ProductImage, ResolvedProductImage},
         resources::{
             CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment,
             Resources, ResourcesFragment,
@@ -324,11 +324,6 @@ impl v1alpha1::AirflowCluster {
         self.spec.cluster_config.volume_mounts.clone()
     }
 
-    /// The name of the role-level load-balanced Kubernetes `Service`
-    pub fn node_role_service_name(&self) -> Option<String> {
-        self.metadata.name.clone()
-    }
-
     /// Retrieve and merge resource configs for role and role groups
     pub fn merged_config(
         &self,
@@ -551,6 +546,7 @@ impl AirflowRole {
     pub fn get_commands(
         &self,
         auth_config: &AirflowClientAuthenticationDetailsResolved,
+        resolved_product_image: &ResolvedProductImage,
     ) -> Vec<String> {
         let mut command = vec![
             format!(
@@ -561,43 +557,79 @@ impl AirflowRole {
             remove_vector_shutdown_file_command(STACKABLE_LOG_DIR),
         ];
 
-        match &self {
-            AirflowRole::Webserver => {
-                // Getting auth commands for AuthClass
-                command.extend(Self::authentication_start_commands(auth_config));
-                command.extend(vec![
+        if resolved_product_image.product_version.starts_with("3.") {
+            // Start-up commands have changed in 3.x.
+            // See https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/upgrading_to_airflow3.html#step-6-changes-to-your-startup-scripts and
+            // https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/setting-up-the-database.html#setting-up-the-database.
+            // `airflow db migrate` is not run for each role so there may be
+            // re-starts of webserver and/or workers (which require the DB).
+            // DB-migrations should be eventually be optional:
+            // See https://github.com/stackabletech/airflow-operator/issues/589.
+            match &self {
+                AirflowRole::Webserver => {
+                    command.extend(Self::authentication_start_commands(auth_config));
+                    command.extend(vec![
+                        "prepare_signal_handlers".to_string(),
+                        container_debug_command(),
+                        "airflow api-server &".to_string(),
+                    ]);
+                }
+                AirflowRole::Scheduler => command.extend(vec![
+                    "airflow db migrate".to_string(),
+                    "airflow users create \
+                        --username \"$ADMIN_USERNAME\" \
+                        --firstname \"$ADMIN_FIRSTNAME\" \
+                        --lastname \"$ADMIN_LASTNAME\" \
+                        --email \"$ADMIN_EMAIL\" \
+                        --password \"$ADMIN_PASSWORD\" \
+                        --role \"Admin\""
+                        .to_string(),
+                    "prepare_signal_handlers".to_string(),
+                    container_debug_command(),
+                    "airflow dag-processor &".to_string(),
+                    "airflow scheduler &".to_string(),
+                ]),
+                AirflowRole::Worker => command.extend(vec![
                     "prepare_signal_handlers".to_string(),
-                    format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"),
-                    "airflow webserver &".to_string(),
-                ]);
+                    container_debug_command(),
+                    "airflow celery worker &".to_string(),
+                ]),
+            }
+        } else {
+            match &self {
+                AirflowRole::Webserver => {
+                    // Getting auth commands for AuthClass
+                    command.extend(Self::authentication_start_commands(auth_config));
+                    command.extend(vec![
+                        "prepare_signal_handlers".to_string(),
+                        container_debug_command(),
+                        "airflow webserver &".to_string(),
+                    ]);
+                }
+                AirflowRole::Scheduler => command.extend(vec![
+                    // Database initialization is limited to the scheduler, see https://github.com/stackabletech/airflow-operator/issues/259
+                    "airflow db init".to_string(),
+                    "airflow db upgrade".to_string(),
+                    "airflow users create \
+                        --username \"$ADMIN_USERNAME\" \
+                        --firstname \"$ADMIN_FIRSTNAME\" \
+                        --lastname \"$ADMIN_LASTNAME\" \
+                        --email \"$ADMIN_EMAIL\" \
+                        --password \"$ADMIN_PASSWORD\" \
+                        --role \"Admin\""
+                        .to_string(),
+                    "prepare_signal_handlers".to_string(),
+                    container_debug_command(),
+                    "airflow scheduler &".to_string(),
+                ]),
+                AirflowRole::Worker => command.extend(vec![
+                    "prepare_signal_handlers".to_string(),
+                    container_debug_command(),
+                    "airflow celery worker &".to_string(),
+                ]),
             }
-
-            AirflowRole::Scheduler => command.extend(vec![
-                // Database initialization is limited to the scheduler, see https://github.com/stackabletech/airflow-operator/issues/259
-                "airflow db init".to_string(),
-                "airflow db upgrade".to_string(),
-                "airflow users create \
-                    --username \"$ADMIN_USERNAME\" \
-                    --firstname \"$ADMIN_FIRSTNAME\" \
-                    --lastname \"$ADMIN_LASTNAME\" \
-                    --email \"$ADMIN_EMAIL\" \
-                    --password \"$ADMIN_PASSWORD\" \
-                    --role \"Admin\""
-                    .to_string(),
-                "prepare_signal_handlers".to_string(),
-                format!(
-                    "containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"
-                ),
-                "airflow scheduler &".to_string(),
-            ]),
-            AirflowRole::Worker => command.extend(vec![
-                "prepare_signal_handlers".to_string(),
-                format!(
-                    "containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"
-                ),
-                "airflow celery worker &".to_string(),
-            ]),
         }
+
         // graceful shutdown part
         command.extend(vec![
             "wait_for_termination $!".to_string(),
@@ -658,6 +690,10 @@ impl AirflowRole {
     }
 }
 
+fn container_debug_command() -> String {
+    format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &")
+}
+
 #[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)]
 pub enum AirflowExecutor {
     /// The celery executor.
@@ -855,17 +891,17 @@ fn default_resources(role: &AirflowRole) -> ResourcesFragment<AirflowStorageConf
     let (cpu, memory) = match role {
         AirflowRole::Worker => (
             CpuLimitsFragment {
-                min: Some(Quantity("500m".into())),
+                min: Some(Quantity("1".into())),
                 max: Some(Quantity("2".into())),
             },
             MemoryLimitsFragment {
-                limit: Some(Quantity("2Gi".into())),
+                limit: Some(Quantity("3Gi".into())),
                 runtime_limits: NoRuntimeLimitsFragment {},
             },
         ),
         AirflowRole::Webserver => (
             CpuLimitsFragment {
-                min: Some(Quantity("500m".into())),
+                min: Some(Quantity("1".into())),
                 max: Some(Quantity("2".into())),
             },
             MemoryLimitsFragment {
@@ -875,11 +911,11 @@ fn default_resources(role: &AirflowRole) -> ResourcesFragment<AirflowStorageConf
         ),
         AirflowRole::Scheduler => (
             CpuLimitsFragment {
-                min: Some(Quantity("500m".to_owned())),
+                min: Some(Quantity("1".to_owned())),
                 max: Some(Quantity("2".to_owned())),
             },
             MemoryLimitsFragment {
-                limit: Some(Quantity("512Mi".to_owned())),
+                limit: Some(Quantity("1Gi".to_owned())),
                 runtime_limits: NoRuntimeLimitsFragment {},
             },
         ),