Run examples on serverless

.github/workflows/run_examples.yml

@@ -0,0 +1,57 @@
+name: Examples
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - "user_guide/**"
+      - "tutorials/**"
+  workflow_dispatch:  # Enables manual trigger
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-workflows:
+    runs-on: ubuntu-latest
+    name: ${{ matrix.config.workflow_directory }}/${{ matrix.config.workflow_file }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "hello_world.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "multiple_tasks.py", workflow_name: "my_parent_wf"}
+          - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "plus_one.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/actors", workflow_file: "pod_template.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "basic.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "model_card.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "partition_keys_input.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "time_partition_input.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "time_partition_runtime.py", workflow_name: "wf"}
+          - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "trigger_on_artifact.py", workflow_name: "downstream_wf"}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run the workflow
+        working-directory: ${{ matrix.config.workflow_directory }}
+        env:
+          UNION_API_KEY: ${{ secrets.UNION_API_KEY }}
+        run: |
+          union run --remote --wait ${{ matrix.config.workflow_file }} ${{ matrix.config.workflow_name }}

requirements.txt

@@ -1,2 +1,6 @@
 pre-commit==3.7.1
-black==24.4.2
+black==24.4.2
+union
+pandas
+pyarrow
+kubernetes

user_guide/core_concepts/actors/serverless/multiple_tasks.py

@@ -28,7 +28,7 @@ def my_child_wf(name: str) -> str:
 
 
 @workflow
-def my_parent_wf(name: str) -> str:
+def my_parent_wf(name: str = "union") -> str:
     a = say_hello(name=name)
     b = my_child_wf(name=a)
     return my_child_wf_lp(name=b)

user_guide/core_concepts/actors/serverless/plus_one.py

@@ -19,4 +19,4 @@ def wf(input: int = 0) -> int:
     a = plus_one(input=input)
     b = plus_one(input=a)
     c = plus_one(input=b)
-    return plus_one(input=c)
+    return plus_one(input=c)

user_guide/core_concepts/artifacts/basic.py

@@ -4,7 +4,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 BasicTaskData = Artifact(

user_guide/core_concepts/artifacts/model_card.py

@@ -5,7 +5,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow", "tabulate"]
 )
 
 BasicArtifact = Artifact(name="my_basic_artifact")

user_guide/core_concepts/artifacts/partition_keys_input.py

@@ -6,7 +6,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 BasicArtifact = Artifact(
@@ -30,5 +30,5 @@ def t1(
 
 
 @workflow
-def wf(dt: datetime, val: str):
+def wf(dt: datetime = datetime.fromtimestamp(1887639468), val: str = "hello"):
     t1(key1=val, dt=dt)

user_guide/core_concepts/artifacts/partition_keys_runtime.py

@@ -6,7 +6,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 BasicArtifact = Artifact(

user_guide/core_concepts/artifacts/time_partition_input.py

@@ -6,7 +6,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 BasicArtifact = Artifact(
@@ -24,5 +24,5 @@ def t1(date: datetime)\
 
 
 @workflow
-def wf(run_date: datetime):
+def wf(run_date: datetime = datetime.fromtimestamp(1887639468)):
     return t1(date=run_date)

user_guide/core_concepts/artifacts/time_partition_runtime.py

@@ -6,7 +6,7 @@
 from typing_extensions import Annotated
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 BasicArtifact = Artifact(

user_guide/core_concepts/artifacts/trigger_on_artifact.py

@@ -7,7 +7,7 @@
 from union.artifacts import OnArtifact
 
 pandas_image = ImageSpec(
-    packages=["pandas==2.2.2"]
+    packages=["pandas==2.2.2", "pyarrow"]
 )
 
 UpstreamArtifact = Artifact(
@@ -36,7 +36,7 @@ def upstream_wf() -> pd.DataFrame:
 )
 
 
-@task
+@task(container_image=pandas_image)
 def downstream_t1():
     print("Downstream task triggered")