Add localstack Breeze integration

Dockerfile.ci

@@ -1208,6 +1208,27 @@ function environment_initialization() {
         ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
     fi
 
+    if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then
+        echo
+        echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}"
+        echo
+
+        # Define LocalStack AWS configuration
+        declare -A localstack_config=(
+            ["AWS_ENDPOINT_URL"]="http://localstack:4566"
+            ["AWS_ACCESS_KEY_ID"]="test"
+            ["AWS_SECRET_ACCESS_KEY"]="test"
+            ["AWS_DEFAULT_REGION"]="us-east-1"
+        )
+
+        # Export each configuration variable and log it
+        for key in "${!localstack_config[@]}"; do
+            export "$key"="${localstack_config[$key]}"
+            echo "  * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}"
+        done
+        echo
+    fi
+
     cd "${AIRFLOW_SOURCES}"
 
     # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags

contributing-docs/testing/integration_tests.rst

@@ -70,6 +70,8 @@ core or provider type of test.
 +--------------+-------------------------------------------------------+
 | keycloak     | Integration for manual testing of multi-team Airflow. |
 +--------------+-------------------------------------------------------+
+| localstack   | Integration that emulates AWS services locally.       |
++--------------+-------------------------------------------------------+
 | mongo        | Integration required for MongoDB hooks.               |
 +--------------+-------------------------------------------------------+
 | mssql        | Integration required for mssql hooks.                 |

dev/breeze/doc/03_developer_tasks.rst

@@ -392,6 +392,29 @@ These are all available flags of ``start-airflow`` command:
   :width: 100%
   :alt: Breeze start-airflow
 
+Running External System Integrations with Breeze
+------------------------------------------------
+
+You can run Airflow alongside external systems in Breeze, such as Kafka, Cassandra, MongoDB, and more.
+
+To start Airflow with an integration, use the following command:
+
+.. code-block:: bash
+
+    breeze --python 3.10 --backend postgres --integration <integration_name>
+
+For example, to run Airflow with Kafka:
+
+.. code-block:: bash
+
+    breeze --python 3.10 --backend postgres --integration kafka
+
+Check the available integrations by running:
+
+.. code-block:: bash
+
+    breeze --integration --help
+
 Launching multiple terminals in the same environment
 ----------------------------------------------------
 

dev/breeze/doc/images/output_shell.txt

@@ -1 +1 @@
-5bdfe1d4afe64fb8bd3e07d5c6e9c2d5
+345a25e9b7fd41a8e1d14d2c275e387d

dev/breeze/doc/images/output_start-airflow.txt

@@ -1 +1 @@
-da22448455ba68de7ee5d2b1bd206265
+c5ecfff87a01e315b8f558798a5dc35c

dev/breeze/doc/images/output_testing_providers-integration-tests.txt

@@ -1 +1 @@
-66a5d51390e696c06b63d6ad296c7784
+c01e190daa309e88d3021a63f3099b52

dev/breeze/src/airflow_breeze/global_constants.py

@@ -72,6 +72,7 @@
     "drill",
     "tinkerpop",
     "kafka",
+    "localstack",
     "mongo",
     "mssql",
     "pinot",
@@ -82,6 +83,7 @@
 ]
 DISABLE_TESTABLE_INTEGRATIONS_FROM_CI = [
     "mssql",
+    "localstack",  # just for local integration testing for now
 ]
 KEYCLOAK_INTEGRATION = "keycloak"
 STATSD_INTEGRATION = "statsd"

dev/breeze/src/airflow_breeze/params/shell_params.py

@@ -429,6 +429,7 @@ def compose_file(self) -> str:
         else:
             integrations = self.integration
         for integration in integrations:
+            get_console().print(f"[info]Adding integration compose file for {integration}[/]")
             compose_file_list.append(DOCKER_COMPOSE_DIR / f"integration-{integration}.yml")
         if "trino" in integrations and "kerberos" not in integrations:
             get_console().print(

scripts/ci/docker-compose/integration-localstack.yml

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+services:
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
+    image: localstack/localstack:4.7
+    labels:
+      breeze.description: "Integration that emulates AWS services locally."
+    ports:
+      - "4566:4566"            # LocalStack Gateway
+      - "4510-4559:4510-4559"  # external services port range
+    environment:
+      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+      - DEBUG=${DEBUG:-0}
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+
+  airflow:
+    environment:
+      - INTEGRATION_LOCALSTACK=true
+    depends_on:
+      - localstack

scripts/docker/entrypoint_ci.sh

@@ -184,6 +184,27 @@ function environment_initialization() {
         ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
     fi
 
+    if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then
+        echo
+        echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}"
+        echo
+
+        # Define LocalStack AWS configuration
+        declare -A localstack_config=(
+            ["AWS_ENDPOINT_URL"]="http://localstack:4566"
+            ["AWS_ACCESS_KEY_ID"]="test"
+            ["AWS_SECRET_ACCESS_KEY"]="test"
+            ["AWS_DEFAULT_REGION"]="us-east-1"
+        )
+
+        # Export each configuration variable and log it
+        for key in "${!localstack_config[@]}"; do
+            export "$key"="${localstack_config[$key]}"
+            echo "  * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}"
+        done
+        echo
+    fi
+
     cd "${AIRFLOW_SOURCES}"
 
     # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags