diff --git a/.github/workflows/run_examples.yml b/.github/workflows/run_examples.yml new file mode 100644 index 00000000..73273c71 --- /dev/null +++ b/.github/workflows/run_examples.yml @@ -0,0 +1,57 @@ +name: Examples + +on: + push: + branches: + - master + pull_request: + paths: + - "user_guide/**" + - "tutorials/**" + workflow_dispatch: # Enables manual trigger + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run-workflows: + runs-on: ubuntu-latest + name: ${{ matrix.config.workflow_directory }}/${{ matrix.config.workflow_file }} + strategy: + fail-fast: false + matrix: + config: + - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "hello_world.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "multiple_tasks.py", workflow_name: "my_parent_wf"} + - {workflow_directory: "user_guide/core_concepts/actors/serverless", workflow_file: "plus_one.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/actors", workflow_file: "pod_template.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "basic.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "model_card.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "partition_keys_input.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "time_partition_input.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "time_partition_runtime.py", workflow_name: "wf"} + - {workflow_directory: "user_guide/core_concepts/artifacts", workflow_file: "trigger_on_artifact.py", workflow_name: "downstream_wf"} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.10' + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run the workflow + working-directory: ${{ matrix.config.workflow_directory }} + env: + UNION_API_KEY: ${{ secrets.UNION_API_KEY }} + run: | + union run --remote --wait ${{ matrix.config.workflow_file }} ${{ matrix.config.workflow_name }} diff --git a/requirements.txt b/requirements.txt index c0f00809..813af609 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,6 @@ pre-commit==3.7.1 -black==24.4.2 \ No newline at end of file +black==24.4.2 +union +pandas +pyarrow +kubernetes \ No newline at end of file diff --git a/user_guide/core_concepts/actors/serverless/multiple_tasks.py b/user_guide/core_concepts/actors/serverless/multiple_tasks.py index 0853b39b..22fc4b03 100644 --- a/user_guide/core_concepts/actors/serverless/multiple_tasks.py +++ b/user_guide/core_concepts/actors/serverless/multiple_tasks.py @@ -28,7 +28,7 @@ def my_child_wf(name: str) -> str: @workflow -def my_parent_wf(name: str) -> str: +def my_parent_wf(name: str = "union") -> str: a = say_hello(name=name) b = my_child_wf(name=a) return my_child_wf_lp(name=b) diff --git a/user_guide/core_concepts/actors/serverless/plus_one.py b/user_guide/core_concepts/actors/serverless/plus_one.py index c1834e8d..b29ea548 100644 --- a/user_guide/core_concepts/actors/serverless/plus_one.py +++ b/user_guide/core_concepts/actors/serverless/plus_one.py @@ -19,4 +19,4 @@ def wf(input: int = 0) -> int: a = plus_one(input=input) b = plus_one(input=a) c = plus_one(input=b) - return plus_one(input=c) + return plus_one(input=c) \ No newline at end of file diff --git a/user_guide/core_concepts/artifacts/basic.py b/user_guide/core_concepts/artifacts/basic.py index 2de15909..888493ce 100644 --- a/user_guide/core_concepts/artifacts/basic.py +++ b/user_guide/core_concepts/artifacts/basic.py @@ -4,7 +4,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) BasicTaskData = Artifact( diff --git a/user_guide/core_concepts/artifacts/model_card.py b/user_guide/core_concepts/artifacts/model_card.py index b8b7e0f8..f626a02a 100644 --- a/user_guide/core_concepts/artifacts/model_card.py +++ b/user_guide/core_concepts/artifacts/model_card.py @@ -5,7 +5,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow", "tabulate"] ) BasicArtifact = Artifact(name="my_basic_artifact") diff --git a/user_guide/core_concepts/artifacts/partition_keys_input.py b/user_guide/core_concepts/artifacts/partition_keys_input.py index 3b4e0d86..4ff19b6b 100644 --- a/user_guide/core_concepts/artifacts/partition_keys_input.py +++ b/user_guide/core_concepts/artifacts/partition_keys_input.py @@ -6,7 +6,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) BasicArtifact = Artifact( @@ -30,5 +30,5 @@ def t1( @workflow -def wf(dt: datetime, val: str): +def wf(dt: datetime = datetime.fromtimestamp(1887639468), val: str = "hello"): t1(key1=val, dt=dt) diff --git a/user_guide/core_concepts/artifacts/partition_keys_runtime.py b/user_guide/core_concepts/artifacts/partition_keys_runtime.py index 04c637f2..6ceaa97f 100644 --- a/user_guide/core_concepts/artifacts/partition_keys_runtime.py +++ b/user_guide/core_concepts/artifacts/partition_keys_runtime.py @@ -6,7 +6,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) BasicArtifact = Artifact( diff --git a/user_guide/core_concepts/artifacts/time_partition_input.py b/user_guide/core_concepts/artifacts/time_partition_input.py index 1f839c09..991f42f1 100644 --- a/user_guide/core_concepts/artifacts/time_partition_input.py +++ b/user_guide/core_concepts/artifacts/time_partition_input.py @@ -6,7 +6,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) BasicArtifact = Artifact( @@ -24,5 +24,5 @@ def t1(date: datetime)\ @workflow -def wf(run_date: datetime): +def wf(run_date: datetime = datetime.fromtimestamp(1887639468)): return t1(date=run_date) diff --git a/user_guide/core_concepts/artifacts/time_partition_runtime.py b/user_guide/core_concepts/artifacts/time_partition_runtime.py index 6929fb70..7bc1441b 100644 --- a/user_guide/core_concepts/artifacts/time_partition_runtime.py +++ b/user_guide/core_concepts/artifacts/time_partition_runtime.py @@ -6,7 +6,7 @@ from typing_extensions import Annotated pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) BasicArtifact = Artifact( diff --git a/user_guide/core_concepts/artifacts/trigger_on_artifact.py b/user_guide/core_concepts/artifacts/trigger_on_artifact.py index 1459721b..295df7f2 100644 --- a/user_guide/core_concepts/artifacts/trigger_on_artifact.py +++ b/user_guide/core_concepts/artifacts/trigger_on_artifact.py @@ -7,7 +7,7 @@ from union.artifacts import OnArtifact pandas_image = ImageSpec( - packages=["pandas==2.2.2"] + packages=["pandas==2.2.2", "pyarrow"] ) UpstreamArtifact = Artifact( @@ -36,7 +36,7 @@ def upstream_wf() -> pd.DataFrame: ) -@task +@task(container_image=pandas_image) def downstream_t1(): print("Downstream task triggered")