From e20b54183129d8b217384d70050d881abdee07b1 Mon Sep 17 00:00:00 2001 From: Grantham Taylor <54340816+granthamtaylor@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:51:55 -0500 Subject: [PATCH 01/27] Add advanced Optuna plugin authoring methods (#3051) * add suggestion bundle support * run pre-commit * update init * simplify bundling setup. Now happens automatically via typing * remove Suggestion type * allow for some suggestions to be defined as kwargs during bundling * organize imports * fix unit tests * remove positional only arg * replace and with or in tests (again) * simplify to suggest over any dictionary * support recursive dictionaries * update recursive method * remove space * fix in-place process error * simplify a tiny bit * add functionality for suggestion callback * fix test * update typing to include Union operator * fixed new unit test * remove ws * clean up callback method * add typing-extensions based concat * add optimize decorator * run pre-commit * add ParamSpec for typing * fix import statements * update docs * Add validation to delay * test runtime validation * whitespace * add unparameterized decorator * add tuple output validation * clean up some tests * whitespace * Clean up README * fix validation, add docstring * update to AsyncPythonFunctionTask * add synchronous objective test --- plugins/flytekit-optuna/README.md | 179 ++++++++++++++++-- .../flytekitplugins/optuna/__init__.py | 4 +- .../flytekitplugins/optuna/optimizer.py | 153 +++++++++++---- plugins/flytekit-optuna/setup.py | 2 +- plugins/flytekit-optuna/tests/__init__.py | 0 .../flytekit-optuna/tests/test_callback.py | 91 +++++++++ .../flytekit-optuna/tests/test_decorator.py | 114 +++++++++++ .../flytekit-optuna/tests/test_imperative.py | 97 ++++++++++ .../flytekit-optuna/tests/test_optimizer.py | 32 ---- .../flytekit-optuna/tests/test_validation.py | 85 +++++++++ plugins/flytekit-optuna/timeline.png | Bin 0 -> 62649 bytes 11 files changed, 666 insertions(+), 91 deletions(-) create mode 100644 plugins/flytekit-optuna/tests/__init__.py create mode 100644 plugins/flytekit-optuna/tests/test_callback.py create mode 100644 plugins/flytekit-optuna/tests/test_decorator.py create mode 100644 plugins/flytekit-optuna/tests/test_imperative.py delete mode 100644 plugins/flytekit-optuna/tests/test_optimizer.py create mode 100644 plugins/flytekit-optuna/tests/test_validation.py create mode 100644 plugins/flytekit-optuna/timeline.png diff --git a/plugins/flytekit-optuna/README.md b/plugins/flytekit-optuna/README.md index 5e247c227f..1a519f91a6 100644 --- a/plugins/flytekit-optuna/README.md +++ b/plugins/flytekit-optuna/README.md @@ -1,24 +1,61 @@ -# Fully Parallelized Flyte Orchestrated Optimizer +# Fully Parallelized Wrapper Around Optuna Using Flyte -WIP Flyte integration with Optuna to parallelize optimization objective function runtime. +## Overview + +This documentation provides a guide to a fully parallelized Flyte plugin for Optuna. This wrapper leverages Flyte's scalable and distributed workflow orchestration capabilities to parallelize Optuna's hyperparameter optimization across multiple trials efficiently. + +![Timeline](timeline.png) + + +## Features + +- **Ease of Use**: This plugin requires no external data storage or experiment tracking. +- **Parallelized Trial Execution**: Enables concurrent execution of Optuna trials, dramatically speeding up optimization tasks. +- **Scalability**: Leverages Flyte’s ability to scale horizontally to handle large-scale hyperparameter tuning jobs. +- **Flexible Integration**: Compatible with various machine learning frameworks and training pipelines. + +## Installation + +- Install `flytekit` +- Install `flytekitplugins.optuna` + +## Getting Started + +### Prerequisites + +- A Flyte deployment configured and running. +- Python 3.9 or later. +- Familiarity with Flyte and asynchronous programming. + +### Define the Objective Function + +The objective function defines the problem to be optimized. It should include the hyperparameters to be tuned and return a value to minimize or maximize. ```python import math import flytekit as fl -from optimizer import Optimizer, suggest - -image = fl.ImageSpec(builder="union", packages=["flytekit==1.15.0b0", "optuna>=4.0.0"]) +image = fl.ImageSpec(packages=["flytekitplugins.optuna"]) @fl.task(container_image=image) async def objective(x: float, y: int, z: int, power: int) -> float: return math.log((((x - 5) ** 2) + (y + 4) ** 4 + (3 * z - 3) ** 2)) ** power +``` + +### Configure the Flyte Workflow + +The Flyte workflow orchestrates the parallel execution of Optuna trials. Below is an example: + +```python +import flytekit as fl +from flytekitplugins.optuna import Optimizer, suggest @fl.eager(container_image=image) -async def train(concurrency: int, n_trials: int): - optimizer = Optimizer(objective, concurrency, n_trials) +async def train(concurrency: int, n_trials: int) -> float: + + optimizer = Optimizer(objective=objective, concurrency=concurrency, n_trials=n_trials) await optimizer( x=suggest.float(low=-10, high=10), @@ -26,20 +63,126 @@ async def train(concurrency: int, n_trials: int): z=suggest.category([-5, 0, 3, 6, 9]), power=2, ) + + print(optimizer.study.best_value) + +``` + +### Register and Execute the Workflow + +Submit the workflow to Flyte for execution: + +```bash +pyflyte register files . +pyflyte run --name train +``` + +### Monitor Progress + +You can monitor the progress of the trials via the Flyte Console. Each trial runs as a separate task, and the results are aggregated by the Optuna wrapper. + +You may access the `optuna.Study` like so: `optimizer.study`. + +Therefore, with `plotly` installed, you may create create Flyte Decks of the study like so: + +```python +import plotly + +fig = optuna.visualization.plot_timeline(optimizer.study) +fl.Deck(name, plotly.io.to_html(fig)) ``` -This integration allows one to define fully parallelized HPO experiments via `@eager` in as little as 20 lines of code. The objective task is optimized via Optuna under the hood, such that one may extract the `optuna.Study` at any time for the purposes of serialization, storage, visualization, or interpretation. +## Advanced Configuration + +### Custom Dictionary Inputs + +Suggestions may be defined in recursive dictionaries: -This plugin provides full feature parity to Optuna, including: +```python +import flytekit as fl +from flytekitplugins.optuna import Optimizer, suggest + +image = fl.ImageSpec(packages=["flytekitplugins.optuna"]) + + +@fl.task(container_image=image) +async def objective(params: dict[str, int | float | str]) -> float: + ... + + +@fl.eager(container_image=image) +async def train(concurrency: int, n_trials: int): + + study = optuna.create_study(direction="maximize") + + optimizer = Optimizer(objective=objective, concurrency=concurrency, n_trials=n_trials, study=study) + + params = { + "lambda": suggest.float(1e-8, 1.0, log=True), + "alpha": suggest.float(1e-8, 1.0, log=True), + "subsample": suggest.float(0.2, 1.0), + "colsample_bytree": suggest.float(0.2, 1.0), + "max_depth": suggest.integer(3, 9, step=2), + "objective": "binary:logistic", + "tree_method": "exact", + "booster": "dart", + } + + await optimizer(params=params) +``` + +### Custom Callbacks + +In some cases, you may need to define the suggestions programmatically. This may be done + +```python +import flytekit as fl +import optuna +from flytekitplugins.optuna import optimize + +image = fl.ImageSpec(packages=["flytekitplugins.optuna"]) + +@fl.task(container_image=image) +async def objective(params: dict[str, int | float | str]) -> float: + ... + +@optimize +def optimizer(trial: optuna.Trial, verbosity: int, tree_method: str): + + params = { + "verbosity:": verbosity, + "tree_method": tree_method, + "objective": "binary:logistic", + # defines booster, gblinear for linear functions. + "booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]), + # sampling according to each tree. + "colsample_bytree": trial.suggest_float("colsample_bytree", 0.2, 1.0), + } + + if params["booster"] in ["gbtree", "dart"]: + # maximum depth of the tree, signifies complexity of the tree. + params["max_depth"] = trial.suggest_int("max_depth", 3, 9, step=2) + + if params["booster"] == "dart": + params["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"]) + params["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"]) + + return objective(params) + + +@fl.eager(container_image=image) +async def train(concurrency: int, n_trials: int): + + optimizer.concurrency = concurrency + optimizer.n_trials = n_trials + + study = optuna.create_study(direction="maximize") + + await optimizer(verbosity=0, tree_method="exact") +``` -- fixed arguments -- multiple suggestion types (`Integer`, `Float`, `Category`) -- multi-objective, with arbitrary objective directions (minimize, maximize) -- pruners -- samplers +## Troubleshooting -# Improvements +Resource Constraints: Ensure sufficient compute resources are allocated for the number of parallel jobs specified. -- This would synergize really well with Union Actors. -- This should also support workflows, but it currently does not. -- Add unit tests, of course. +Flyte Errors: Refer to the Flyte logs and documentation to debug workflow execution issues. diff --git a/plugins/flytekit-optuna/flytekitplugins/optuna/__init__.py b/plugins/flytekit-optuna/flytekitplugins/optuna/__init__.py index 55cd84fc8a..9804fb94a8 100644 --- a/plugins/flytekit-optuna/flytekitplugins/optuna/__init__.py +++ b/plugins/flytekit-optuna/flytekitplugins/optuna/__init__.py @@ -1,3 +1,3 @@ -from .optimizer import Optimizer, suggest +from .optimizer import Optimizer, optimize, suggest -__all__ = ["Optimizer", "suggest"] +__all__ = ["Optimizer", "optimize", "suggest"] diff --git a/plugins/flytekit-optuna/flytekitplugins/optuna/optimizer.py b/plugins/flytekit-optuna/flytekitplugins/optuna/optimizer.py index 88a4f05f62..924dc3694a 100644 --- a/plugins/flytekit-optuna/flytekitplugins/optuna/optimizer.py +++ b/plugins/flytekit-optuna/flytekitplugins/optuna/optimizer.py @@ -1,12 +1,14 @@ import asyncio import inspect +from copy import copy, deepcopy from dataclasses import dataclass from types import SimpleNamespace -from typing import Any, Optional, Union +from typing import Any, Awaitable, Callable, Optional, Union + +from typing_extensions import Concatenate, ParamSpec import optuna -from flytekit import PythonFunctionTask -from flytekit.core.workflow import PythonFunctionWorkflow +from flytekit.core.python_function_task import AsyncPythonFunctionTask from flytekit.exceptions.eager import EagerException @@ -45,52 +47,82 @@ class Category(Suggestion): suggest = SimpleNamespace(float=Float, integer=Integer, category=Category) +P = ParamSpec("P") + +Result = Union[float, tuple[float, ...]] + +CallbackType = Callable[Concatenate[optuna.Trial, P], Union[Awaitable[Result], Result]] + @dataclass class Optimizer: - objective: Union[PythonFunctionTask, PythonFunctionWorkflow] + objective: Union[CallbackType, AsyncPythonFunctionTask] concurrency: int n_trials: int study: Optional[optuna.Study] = None + delay: int = 0 + + """ + Optimizer is a class that allows for the distributed optimization of a flytekit Task using Optuna. + + Args: + objective: The objective function to be optimized. This can be a AsyncPythonFunctionTask or a callable. + concurrency: The number of trials to run concurrently. + n_trials: The number of trials to run in total. + study: The study to use for optimization. If None, a new study will be created. + delay: The delay in seconds between starting each trial. Default is 0. + """ + + @property + def is_imperative(self) -> bool: + return isinstance(self.objective, AsyncPythonFunctionTask) def __post_init__(self): if self.study is None: self.study = optuna.create_study() - if (not isinstance(self.concurrency, int)) or (self.concurrency < 0): + if (not isinstance(self.concurrency, int)) or (not self.concurrency > 0): raise ValueError("concurrency must be an integer greater than 0") - if (not isinstance(self.n_trials, int)) or (self.n_trials < 0): + if (not isinstance(self.n_trials, int)) or (not self.n_trials > 0): raise ValueError("n_trials must be an integer greater than 0") if not isinstance(self.study, optuna.Study): raise ValueError("study must be an optuna.Study") - # check if the objective function returns the correct number of outputs - if isinstance(self.objective, PythonFunctionTask): - func = self.objective.task_function - elif isinstance(self.objective, PythonFunctionWorkflow): - func = self.objective._workflow_function - else: - raise ValueError("objective must be a PythonFunctionTask or PythonFunctionWorkflow") + if not isinstance(self.delay, int) or (not self.delay >= 0): + raise ValueError("delay must be an integer greater than or equal to 0") - signature = inspect.signature(func) + if self.is_imperative: + signature = inspect.signature(self.objective.task_function) - if signature.return_annotation is float: - if len(self.study.directions) != 1: - raise ValueError("the study must have a single objective if objective returns a single float") + if signature.return_annotation is float: + if len(self.study.directions) != 1: + raise ValueError("the study must have a single objective if objective returns a single float") - elif isinstance(args := signature.return_annotation.__args__, tuple): - if len(args) != len(self.study.directions): - raise ValueError("objective must return the same number of directions in the study") + elif hasattr(signature.return_annotation, "__args__"): + args = signature.return_annotation.__args__ + if len(args) != len(self.study.directions): + raise ValueError("objective must return the same number of directions in the study") - if not all(arg is float for arg in args): + if not all(arg is float for arg in args): + raise ValueError("objective function must return a float or tuple of floats") + + else: raise ValueError("objective function must return a float or tuple of floats") else: - raise ValueError("objective function must return a float or tuple of floats") + if not callable(self.objective): + raise ValueError("objective must be a callable or a AsyncPythonFunctionTask") + + signature = inspect.signature(self.objective) - async def __call__(self, **inputs: Any): + if "trial" not in signature.parameters: + raise ValueError( + "objective function must have a parameter called 'trial' if not a AsyncPythonFunctionTask" + ) + + async def __call__(self, **inputs: P.kwargs): """ Asynchronously executes the objective function remotely. Parameters: @@ -101,31 +133,28 @@ async def __call__(self, **inputs: Any): semaphore = asyncio.Semaphore(self.concurrency) # create list of async trials - trials = [self.spawn(semaphore, **inputs) for _ in range(self.n_trials)] + trials = [self.spawn(semaphore, deepcopy(inputs)) for _ in range(self.n_trials)] # await all trials to complete await asyncio.gather(*trials) - async def spawn(self, semaphore: asyncio.Semaphore, **inputs: Any): + async def spawn(self, semaphore: asyncio.Semaphore, inputs: dict[str, Any]): async with semaphore: + await asyncio.sleep(self.delay) + # ask for a new trial trial: optuna.Trial = self.study.ask() - suggesters = { - Float: trial.suggest_float, - Integer: trial.suggest_int, - Category: trial.suggest_categorical, - } - - # suggest inputs for the trial - for key, value in inputs.items(): - if isinstance(value, Suggestion): - suggester = suggesters[type(value)] - inputs[key] = suggester(name=key, **vars(value)) - try: + result: Union[float, tuple[float, ...]] + # schedule the trial - result: Union[float, tuple[float, ...]] = await self.objective(**inputs) + if self.is_imperative: + result = await self.objective(**process(trial, inputs)) + + else: + out = self.objective(trial=trial, **inputs) + result = out if not inspect.isawaitable(out) else await out # tell the study the result self.study.tell(trial, result, state=optuna.trial.TrialState.COMPLETE) @@ -133,3 +162,51 @@ async def spawn(self, semaphore: asyncio.Semaphore, **inputs: Any): # if the trial fails, tell the study except EagerException: self.study.tell(trial, state=optuna.trial.TrialState.FAIL) + + +def optimize( + objective: Optional[Union[CallbackType, AsyncPythonFunctionTask]] = None, + concurrency: int = 1, + n_trials: int = 1, + study: Optional[optuna.Study] = None, +): + if objective is not None: + if callable(objective) or isinstance(objective, AsyncPythonFunctionTask): + return Optimizer( + objective=objective, + concurrency=concurrency, + n_trials=n_trials, + study=study, + ) + + else: + raise ValueError("This decorator must be called with a callable or a flyte Task") + else: + + def decorator(objective): + return Optimizer(objective=objective, concurrency=concurrency, n_trials=n_trials, study=study) + + return decorator + + +def process(trial: optuna.Trial, inputs: dict[str, Any], root: Optional[list[str]] = None) -> dict[str, Any]: + if root is None: + root = [] + + suggesters = { + Float: trial.suggest_float, + Integer: trial.suggest_int, + Category: trial.suggest_categorical, + } + + for key, value in inputs.items(): + path = copy(root) + [key] + + if isinstance(inputs[key], Suggestion): + suggester = suggesters[type(value)] + inputs[key] = suggester(name=(".").join(path), **vars(value)) + + elif isinstance(value, dict): + inputs[key] = process(trial=trial, inputs=value, root=path) + + return inputs diff --git a/plugins/flytekit-optuna/setup.py b/plugins/flytekit-optuna/setup.py index 82e3e2f880..3bc7285d1f 100644 --- a/plugins/flytekit-optuna/setup.py +++ b/plugins/flytekit-optuna/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.15.0", "optuna>=4.0.0,<5.0.0"] +plugin_requires = ["flytekit>=1.15.0", "optuna>=4.0.0,<5.0.0", "typing-extensions>=4.10,<5.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-optuna/tests/__init__.py b/plugins/flytekit-optuna/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-optuna/tests/test_callback.py b/plugins/flytekit-optuna/tests/test_callback.py new file mode 100644 index 0000000000..a7a6eaf768 --- /dev/null +++ b/plugins/flytekit-optuna/tests/test_callback.py @@ -0,0 +1,91 @@ +from typing import Union + +import asyncio +from typing import Union +import optuna + +import flytekit as fl +from flytekitplugins.optuna import Optimizer + + +def test_callback(): + + + @fl.task + async def objective(letter: str, number: Union[float, int], other: str, fixed: str) -> float: + + loss = len(letter) + number + len(other) + len(fixed) + + return float(loss) + + def callback(trial: optuna.Trial, fixed: str): + + letter = trial.suggest_categorical("booster", ["A", "B", "BLAH"]) + + if letter == "A": + number = trial.suggest_int("number_A", 1, 10) + elif letter == "B": + number = trial.suggest_float("number_B", 10., 20.) + else: + number = 10 + + other = trial.suggest_categorical("other", ["Something", "another word", "a phrase"]) + + return objective(letter, number, other, fixed) + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + + study = optuna.create_study(direction="maximize") + + optimizer = Optimizer(callback, concurrency=concurrency, n_trials=n_trials, study=study) + + await optimizer(fixed="hello!") + + return float(optimizer.study.best_value) + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) + + +def test_async_callback(): + + @fl.task + async def objective(letter: str, number: Union[float, int], other: str, fixed: str) -> float: + + loss = len(letter) + number + len(other) + len(fixed) + + return float(loss) + + async def callback(trial: optuna.Trial, fixed: str): + + letter = trial.suggest_categorical("booster", ["A", "B", "BLAH"]) + + if letter == "A": + number = trial.suggest_int("number_A", 1, 10) + elif letter == "B": + number = trial.suggest_float("number_B", 10., 20.) + else: + number = 10 + + other = trial.suggest_categorical("other", ["Something", "another word", "a phrase"]) + + return await objective(letter, number, other, fixed) + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + + study = optuna.create_study(direction="maximize") + + optimizer = Optimizer(callback, concurrency=concurrency, n_trials=n_trials, study=study) + + await optimizer(fixed="hello!") + + return float(optimizer.study.best_value) + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) diff --git a/plugins/flytekit-optuna/tests/test_decorator.py b/plugins/flytekit-optuna/tests/test_decorator.py new file mode 100644 index 0000000000..e2ec0e7026 --- /dev/null +++ b/plugins/flytekit-optuna/tests/test_decorator.py @@ -0,0 +1,114 @@ +from typing import Union +import math + +import asyncio +from typing import Union +import optuna + +import flytekit as fl +from flytekitplugins.optuna import optimize, suggest + + +def test_local_exec(): + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + + @optimize(concurrency=concurrency, n_trials=n_trials) + @fl.task + async def optimizer(x: float, y: int, z: int, power: int) -> float: + return (((x - 5) ** 2) + (y + 4) ** 4 + (3 * z - 3) ** 2) ** power + + await optimizer( + x=suggest.float(low=-10, high=10), + y=suggest.integer(low=-10, high=10), + z=suggest.category([-5, 0, 3, 6, 9]), + power=2, + ) + + return optimizer.study.best_value + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) + + +def test_callback(): + + + @fl.task + async def objective(letter: str, number: Union[float, int], other: str, fixed: str) -> float: + + loss = len(letter) + number + len(other) + len(fixed) + + return float(loss) + + @optimize(n_trials=10, concurrency=2) + def optimizer(trial: optuna.Trial, fixed: str): + + letter = trial.suggest_categorical("booster", ["A", "B", "BLAH"]) + + if letter == "A": + number = trial.suggest_int("number_A", 1, 10) + elif letter == "B": + number = trial.suggest_float("number_B", 10., 20.) + else: + number = 10 + + other = trial.suggest_categorical("other", ["Something", "another word", "a phrase"]) + + return objective(letter, number, other, fixed) + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + + await optimizer(fixed="hello!") + + return float(optimizer.study.best_value) + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) + + +def test_unparameterized_callback(): + + + @fl.task + async def objective(letter: str, number: Union[float, int], other: str, fixed: str) -> float: + + loss = len(letter) + number + len(other) + len(fixed) + + return float(loss) + + @optimize + def optimizer(trial: optuna.Trial, fixed: str): + + letter = trial.suggest_categorical("booster", ["A", "B", "BLAH"]) + + if letter == "A": + number = trial.suggest_int("number_A", 1, 10) + elif letter == "B": + number = trial.suggest_float("number_B", 10., 20.) + else: + number = 10 + + other = trial.suggest_categorical("other", ["Something", "another word", "a phrase"]) + + return objective(letter, number, other, fixed) + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + + optimizer.n_trials = n_trials + optimizer.concurrency = concurrency + + await optimizer(fixed="hello!") + + return float(optimizer.study.best_value) + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) diff --git a/plugins/flytekit-optuna/tests/test_imperative.py b/plugins/flytekit-optuna/tests/test_imperative.py new file mode 100644 index 0000000000..2c08e3b3dd --- /dev/null +++ b/plugins/flytekit-optuna/tests/test_imperative.py @@ -0,0 +1,97 @@ +from typing import Union + +import asyncio +from typing import Union + +import optuna +import flytekit as fl +from flytekitplugins.optuna import Optimizer, suggest + + + +def test_local_exec(): + + + @fl.task + async def objective(x: float, y: int, z: int, power: int) -> float: + return (((x - 5) ** 2) + (y + 4) ** 4 + (3 * z - 3) ** 2) ** power + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + optimizer = Optimizer(objective, concurrency=concurrency, n_trials=n_trials) + + await optimizer( + x=suggest.float(low=-10, high=10), + y=suggest.integer(low=-10, high=10), + z=suggest.category([-5, 0, 3, 6, 9]), + power=2, + ) + + return optimizer.study.best_value + + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) + +def test_tuple_out(): + + @fl.task + async def objective(x: float, y: int, z: int, power: int) -> tuple[float, float]: + + y0 = (((x - 5) ** 2) + (y + 4) ** 4 + (3 * z - 3) ** 2) ** power + y1 = (((x - 2) ** 4) + (y + 1) ** 2 + (4 * z - 1)) + + return y0, y1 + + + @fl.eager + async def train(concurrency: int, n_trials: int): + optimizer = Optimizer( + objective=objective, + concurrency=concurrency, + n_trials=n_trials, + study=optuna.create_study(directions=["maximize", "maximize"]) + ) + + await optimizer( + x=suggest.float(low=-10, high=10), + y=suggest.integer(low=-10, high=10), + z=suggest.category([-5, 0, 3, 6, 9]), + power=2, + ) + + asyncio.run(train(concurrency=2, n_trials=10)) + + +def test_bundled_local_exec(): + + @fl.task + async def objective(suggestions: dict[str, Union[int, float]], z: int, power: int) -> float: + + # building out a large set of typed inputs is exhausting, so we can just use a dict + + x, y = suggestions["x"], suggestions["y"] + + return (((x - 5) ** 2) + (y + 4) ** 4) ** power + + + @fl.eager + async def train(concurrency: int, n_trials: int) -> float: + optimizer = Optimizer(objective, concurrency=concurrency, n_trials=n_trials) + + suggestions = { + "x": suggest.float(low=-10, high=10), + "y": suggest.integer(low=-10, high=10), + } + + await optimizer( + suggestions=suggestions, + z=suggest.category([-5, 0, 3, 6, 9]), + power=2, + ) + + return optimizer.study.best_value + loss = asyncio.run(train(concurrency=2, n_trials=10)) + + assert isinstance(loss, float) diff --git a/plugins/flytekit-optuna/tests/test_optimizer.py b/plugins/flytekit-optuna/tests/test_optimizer.py deleted file mode 100644 index 33150fd03d..0000000000 --- a/plugins/flytekit-optuna/tests/test_optimizer.py +++ /dev/null @@ -1,32 +0,0 @@ -import asyncio -import math - -import flytekit as fl -from flytekitplugins.optuna import Optimizer, suggest - - -image = fl.ImageSpec(builder="union", packages=["flytekit==1.15.0b0", "optuna>=4.0.0"]) - -@fl.task(container_image=image) -async def objective(x: float, y: int, z: int, power: int) -> float: - return math.log((((x - 5) ** 2) + (y + 4) ** 4 + (3 * z - 3) ** 2)) ** power - - -@fl.eager(container_image=image) -async def train(concurrency: int, n_trials: int) -> float: - optimizer = Optimizer(objective, concurrency, n_trials) - - await optimizer( - x=suggest.float(low=-10, high=10), - y=suggest.integer(low=-10, high=10), - z=suggest.category([-5, 0, 3, 6, 9]), - power=2, - ) - - return optimizer.study.best_value - -def test_local_exec(): - - loss = asyncio.run(train(concurrency=2, n_trials=10)) - - assert isinstance(loss, float) diff --git a/plugins/flytekit-optuna/tests/test_validation.py b/plugins/flytekit-optuna/tests/test_validation.py new file mode 100644 index 0000000000..030b9988ca --- /dev/null +++ b/plugins/flytekit-optuna/tests/test_validation.py @@ -0,0 +1,85 @@ +import flytekit as fl +import pytest + +from flytekitplugins.optuna import Optimizer + + +@fl.task +async def objective(x: float, y: int, z: int, power: int) -> float: + return 1.0 + +def test_concurrency(): + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=-1, n_trials=10) + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=0, n_trials=10) + + with pytest.raises(ValueError): + Optimizer(objective, concurrency="abc", n_trials=10) + + +def test_n_trials(): + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials=-10) + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials=0) + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials="abc") + + + +def test_study(): + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials=10, study="abc") + + + +def test_delay(): + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials=10, delay=-1) + + with pytest.raises(ValueError): + Optimizer(objective, concurrency=3, n_trials=10, delay="abc") + + + +def test_objective(): + + @fl.workflow + def workflow(x: int, y: int, z: int, power: int) -> float: + return 1.0 + + + with pytest.raises(ValueError): + Optimizer(workflow, concurrency=3, n_trials=10) + + @fl.task + async def mistyped_objective(x: int, y: int, z: int, power: int) -> str: + return "abc" + + with pytest.raises(ValueError): + Optimizer(mistyped_objective, concurrency=3, n_trials=10) + + @fl.task + def synchronous_objective(x: int, y: int, z: int, power: int) -> float: + return 1.0 + + with pytest.raises(ValueError): + Optimizer(synchronous_objective, concurrency=3, n_trials=10) + + + +def test_callback(): + + def callback(value: float): + return 1.0 + + with pytest.raises(ValueError): + Optimizer(callback, concurrency=3, n_trials=10) diff --git a/plugins/flytekit-optuna/timeline.png b/plugins/flytekit-optuna/timeline.png new file mode 100644 index 0000000000000000000000000000000000000000..473ac0a4f914414e06460e076e078dbb642a1b78 GIT binary patch literal 62649 zcmeFac|6r!_dkrpQQ;sF;iMw7Q06HqB6E>h$f<;oA>&Ddkd#6hqfk+jF>|CshB6hQ z0m(ei&a<{dxqkQk{eAE2H#~nl&(-VI<F#Gm1;%>99Q+^#P$e-u-+@IErS$DwSNaw)wjNMbkMmjpp zuP>T;hEu#866lN`j+P}Z_SX2-#>l!1bC5M^JHP(6&0w9ssc!MkeLMa4?32E%{ELcG zY5j##&Yw=6o+pmEx>EYvW-NMny061!PCZW2?2lh0X5+`!d&-f8;R8R;cov!l(v$le zj#b?9CJPknIB~kz=!(_)&y1|NijB5XDg*&rsUw8xcW1v@f5PUHCG78Z-nEr1)$EE~ zl7A%m-o`nc+m_MG++>YgvGYAWT0JT;8=ebgF{lWBv`b!02p^SvpSAsnu%tCcpod$8 zpYTfV9G&|d@op&5%-Q8ZfI^zp!;^>iTid!^@hrF_FKv3|5q3K019uXx`Ql7Km`Ltb z)`vESRhTwUr80=zpmm{g-R{=ovaWplzKc>ux0$xy|MBs|<+5#oUpFYH(2PruV*6DJ zb;4LSr(}H+yu0nm#O3CP8m)r6{6Yn8&fl56fP1fH$v0_Xw?XQ#k7l~4xB|_I4HXgV zUsb3DHsRc=szWy#ehv|?u{lKdS?$5|%o(BWq0E8%DK^g;DZkt)b?lVIy=X}(3wxL3 zFs4Tp)MMh`RhVbXj6F;}7o3D!Z3%&QL{oRXWoO?b#ZrtDRomm9W`a8$sQ=3w+e=-+ zU-@nGIjyV@oRTa*F1__|2{aqu@a?wsXoYtkj(N`GrIezKIRmeG)N9({V-*1+>lplL zDahBUSshsyuDk`CGu=G>N>)`{w2Ok~Eg3~ustwhlo$tI3jIdE}D2vC^D0S55k$FBN z+pM(IkA0FSbJ5k)k>Gt`pT~xC!We4l5p#a7U2J0!m^h__GxWozMxLS_+l*`%iE&&u zCE|i_5A5FBmvHFHT*9tBfvjqqc2O}6I;pg?OjqvcYT`P^5_8RquJ(I@{8zsF0V9D< zvGwYC^6ExAXZfo3BryG=7I3*aWi9MT%x6s zF^OQV)c))jT}H31a_W#9Pm+y%rsAim-kYbVPCKaZXPOeDxKz_lsmz|FpIjJ-UsRcz z&7dWIVl)Zls4#x6-|9KQdW@dch^g5>+v<(22mZYL&1X-Q9iNYRWZn-Q;aZ}KPZJFH ztle8HUu&`1XZGT$Om_)O?Je4o7;&-jt%5VESWzkNxQ|X(f?os+kYx@Bf18}-Ik@yi z$+fN6Wt>P>nsUn0vg@b2a2k#OBB#?l*KF-4v9BpA%>FKiJoMBF7t3u~D7-aN%VSTH zY5K3vxIBK2e5jOtCIE+}N>q|lqhhj}xa$smLmaS+6G zx3Gm)CNb4-a@}AYxFt!WJ{!;cl098T`#wuUKc07vV(<-Fa!}uilTkY=D_V^xSkN8oqeeo_nE3IdX-xgE6}}ya2^Dubly^~OlrrY3cA4Gx;L*t$twK7-vS;7u zM->*>2?s^FrLcvVZLf^>Q(5ufzoetpswJfhavGQH!nmi{N?653M zlGtfRZARI^8`lI~_2q&|Lp@5zP7m8lz!_C9`k^h?SGt;zep4n5w|6jzlTnsWG&MyhNo@1xvD zj(CZCd*8=L?5|D{x_|Ve)<@2Ih17L~XpKYW#g)$I*tZ>D6>-g{PAO{wowJ6kKH zDvaz4YiF{~J-hbwT8VnZJN|Gc9{FTfeGC2jef%GJKK5?npbEZOZd8p2q27A;@;&wU zIZ@fsKKpX_Xp4%7b)NC)jjznam&gKc2pR^huM~ z8TS62{p*`2o4@Rv+qW%xe>7XPSB`{GB6gx2+*v!xw z^CiY5CRe(X&WV)o$!z_sBElEqud|BNf}LTN7+I-eL3%P4MuwFVGJ$3 z8Rvy?>zc3U9!vXJaaz52W&G;V*K=R{%ehor52l3AP z1(JpiwH?sDrog5WaxL^xpeLKGnA=JFGuz}H^xB{1xsQ^MYJ4B&Y2{I4y2sSS^oI8s zZ?X0UP2-!5(ry*4$E`(p*m*XFv+JqxUE^!hpL?_Zx{3VLGlgw4w?-v-S~a8geO)+E zeodO`hPGaOMXamnov?eiUkXfalngR?J*fTt!O`T&(bmGfg)H8FOZ>h!N)Ki19})b* z>!W(X{fc&FRY;s>NzGV|`F4{$^BH~ti8K1m^8={^9G})nt4n*i9Cg-rxe=Ik;8SA6 zJD%{ABSTo(K#hu6{let5xDUKEk@j^eV?00h{y0<4HM{Ro`UFc~IYsBipo{eSOevf( z$6z|36k-pv}-a(_id5(QN_t)T+w(1%!zk8! zhk0g9g*Dx;DxWBScwX(x#LMEjnz`C5KdyxCusU>)y@;Kj>%7d%?@^zg2Yr`$6B(Bf zvs+-gV{>P6Upn@-`<_qfs_iXs8DKcv{jr}%F7 zIX>Ssbmn{Rh~qSsGc{F(DUIQhfPYy~a>}}q3)Ey;He^3Xybo?oks`j^m?-t4ieX0E z=-k2EEbe5dzY@JPMHUM3X-bRFj?Ao|Y!4(a&kT8=zv4xTAgq_ z?<&vDjVAj2heW5X$EiOvIk>DW3l=DXej~C61pfEf&{Yn-mC-rnVQYW)t0RP<$f6-6)Yx_g2awwFH zOo>eWfUN@&s(2d zl(2h~^Vyrh?b~TLQ1Igz%C>Uj7?qQvFjSatm$!buWz~E@hr52agwM05zaC4Ue>vZ- z-89?bSTbA?Z%4c!w54;lc6w-0u7qfp{&32#sp?vjUyvULi&OF^Bd4JJI}cAN1cIoF zssC0VQsdHeiXf`o!!wS5Z^GryrJeXL*}qRLCB_~z!%n+Tu#o%jcmA?*H=8Swne6ZP zq})fj`JYMu8`1iQz5l`S`$v%ednWxONdG;L{?SPPfsOk|kp5#qdR#m=&}n8kDR-3t zr<7nzaH&&x5#v5nGnOx6Zm1Q5_tblVexXeWcJOkOcAiKc>ptT!z*O1{)*Nrtm8L>n zJy5|v3|#-Gcq%$dR^lwa(I;dl?o`R>7U*X7M3e8&7bod z&Rcr>+>LtgR&ZNnn5FzCiS8aV`y`1(df8nW`{3ohEqOC;SF5unw+R_qxM%fA(;o6C zBXF*-Rp<|K{+fTairM02al31KNl43g_pZVwKd$liCSvU4%YK__4<1%-u-s|%h7ni) zz&}wr{5fk7xJUiqw)Af{2i_z|ldA^waPPswJUN)%Z1!JA)}bjGbtLfobmh>(@~VGg z86Be|YRm_XlfQ{>`rY{F7`18u-3dd9pEw%v}Z8W*9+iiwf|-@`zGH zDGLGSJq!LvR_u*F4ECpqQbWSmKIB3JBVYzBpfT~-am7dY zu)Uv-c=55>zZ$tkMwkHV`gY{t$SKNTzKhg&Zwg91SjL0U7H3M#POOfE8a9y`gH;nq zz=!QK4raiG9k8rPEc5EruZmwd$GH0pnqHW2rzu#w=6t>v4Hg%s(M6+ptZLBF!jV@s z;6jA1ww^@by><#00X9MZ?j%agB^DhC4A=ejFe)%KOwO{O0f#kKYQVXAU*T0sSO+|y z+n>WtPGJQky*g;;1cJlB`T{>Pk#c*iIoFlq)R9$(l+d$YVdkb@aS!KYrGZ4M-PXyS zssX9*ghX=_%P5Q#4*M&4z-S{3#=y`Yu+(?!ayZE;?!&nI-`IJ;xUfLd-eQjd0rsv` zKmZeT(Ylz!gZKuk^HwlEj<8MtfLCMy&*{j zT8jq3PV7#j+XdbB!Z@dI(}5$abRHgmJ)qkOk4NG0aSYv#RVNj11f9P+T?|deV&JAkFL5)D9(FCdkc?i5e|`C3;wm`~&+#6qJb@cC#Hg zQrk|6ncz;q*K2&3f@zT0U|?}09RCph56Axzm46QBKU(Dz$G<#+4)#nDMl?1XT)k9zoB>E zob$(XemaIYTK7wd%6V_Ma2YjNo)g!`;;@F`m^!%efX_@1Uaf+2&MtC_J>XoQ=s9^B z;vEKXu6fTyGvct?(9rxO506rU2w_h!m5=)82=Gw9V@$mZ%DX?ahekYkCM#3BUJKl2 zGC~=+wgG(~faQ z0Vwdk`#F$|AP4kp^0vdrk{bP>wB1P7Up6&3ci5&i|*}>rDXP|iNl?^8Z2c-{S zws7Zy5a_oN0w+Pu9>iKwkv@XyD#Mc z^Ueb4IpcdcpyOs*Z)%vJ{wb`h;8PSDl^42r)1#N9s4*&(j8F{k=4y#Qrh=$Ays0-!h0NQE3;222Xe6eG~B=>B;zd9aih>iFnb7G#45yUKc{w*9J8QKn7@mf3zE) zAu+uNz0X{Wx(dBbKn!y(%v^&M+qmiJJY;A{P3|9w}5=&6QvQSi$*TVc4f55g7L2 zmN-9~AkEKn6qNg6xtD^fGbk~xyWucd{XE8_gbM-AM5S&5SRw_6iKTKp3k_lROM&mg zpdp-+`g1eKcu7pb>;$|>ZJUG26VM@>D~*zNvxV&HH>IF7M%dFHW^r^1Re+P_uRy_J zvOqo68*}$;2MIm~>`3SIMaY8&TIWNnEzn{B`n8_=#z7JsSj!CjZd5!ZgOI3v>2^en zZiA(|(hVjY_Ao5T!1T&4r34iih)yLJ)T0$}=G6gP9R%WmT9@!R0}b~>!vmUEb}A)U zL&MPWT$xp=L=yDPiQV`}L&AO0N;cI7OTpv4AX$m(CXgAYxKX47&dLv#+S7E$U z(ZWH>Ijen8m?cu>YHj=IDb)auogHvWNc5y1^jF#db8X&Ze36WRG`D&{_7-xAQ?R%X z2Tn0iP%^^wJ2y2Qro=pe+0~d+Y<~->0|?&8(I{B6<%aH?Lmt&_eoqRA-VX~(n)=xc zURg&?fBUL;|6;XE+|-P7?bjDAjz=YoQbu4RIvAd$p)bLfVmRWz(yWEP%z=yEm$v+S zDpwe484V2BhgA#(h(~lo|IF3mc^|IbbzYy?t9^#f-xzzwId!8tzct4EaHBa)XR80u1@O zBMbKC8k{1=t7rBC7zpC|Ah3aP2s7t4LpNQcH;NOF>U@rP5~&rAb&NMh zMn0RX3T_dW^IhHE8-rPjx zw8G|;#cREVFBg{lmcqvfP5aByeu81Zt$x? zZeLDx8aFl3)OvRg`{tQWy=sYc^yylx3XWea?;iP-%jdY5-6)MjT`-b z!#sX}gJtZijeo%+e}kd8e3*XBKmE{JAd{rhvf0Qf_<%=x6?mH8%s2~_wJ9w~kx-`v^#quXdb`O@ zGD1C!bpN}!967~1kcC^+_VN^zuRu*-x=|0akyO*uCV+6kYJ_ng)cAce?a_8peS4z| zv>mjH?Wyma@8Y`CTeNV!``4ML0^JqYoGZ85-RAOKn7;kSw-bB|O(uc~?7a@`eVI-O zGA2i0@@UaQ1eaRF?BHHCDFhD5`9-{W=k8!*)Bu-s)T8P0N~E_budFap05%+Mivu?J;V zv_NmoaQ;^>Nyf?s^#9wa(_XTCY*~@-PVvq)TKMHG1Rs(w@*Xbd(u;xO3#+H^1^05P$$a6UOk9>e$ZN2oh z8xKrNY6x{7xh}~mL_hbGOPIxCg1P%k$r##&Oy5r~7E@Qo*oWsWJ;&=74f2PJO56s0 zR0Ga)ia>rv34|peEM3iC2@g!DAKe44}4(+k=;qxavJnG?136X(yrh7 zJO?ba{kqZbyAdH7p?*JKn2*ejnG6%1W|5%G=1}`s=XVvlR8cF8jgEP7%^1yA$yyk6 z%=Sdy4xpw_qXjqU|FSYG>P!CKZRa;S*zHp?|8*?a&%OKed~cjro?KV-keTnVM>j-z z(~FBIxj>oi`oE*huGJNeJw8)4+0v;aod12$z36y@rQsGwMqC(pU$|2l&D=`3>u@IS z?H1baue2XF^bn<bfMA=$AB8C~q_gul_P6w)K#g>)J$qy~)1IU&orI_ld`3~e z+-hjGHbk_6p~H0xP!zE?pg_xBi8A4EvugG#|5h7UK!DlkcpLEGyuNBb!>k%mWK#$> z=pc|A^5`@0R};p(`yp%NT5>ZHxf+cMKrS4zx3`ncK#76(8Mf!BEa(lA763g#rwK~? z|8!tS8lM)nKw_I#VgvXOm-AcrgQ&dTdBl?uLV(<@`q|*t-39K7Jv_xiLAi!p6q84! z`T)70*u(ga6hL$b!3ECxvD@!DFqF(%@xJa3gMdhrwJ_mCAAw)=pj{Z!X{E1Vj*a1- zXqn-_3C)9pC_-Bs!6WC1=7k1qk0_EW(O7DT0!e&fL;6)J=0Fl3+s+Lx&yKD-A6sE3 z#F9_*{zVx6kPA+Cn2_!~kP9&aPXC@8U=m46*@^+`W0 zbi1;WQ(Q-U7U2h^)# zC^3zol~eEk+yp!c0enX^EexV7Z_?8F@pW5_dUo1p`au5DIU+2U#*9z+x(I5e@nm9 zZ8PFyiE&9a^(uev{JkNXC;uY?m2eq;gmOC$!K<}$Nv59fZ!RkS2uH6)@4^yo8ihe| ztf+*CUT`+e9>}ujt;D}$a7KQ-Z$1fVAjsd_n@K{TdHCQciyNANw|5AdYe(!mWddla zp7~5+j-7;A5{1AMmvmphh2_Zq`i2>o3lo-fu}7=EiwJeov0V4Pvk{yfqz(ji`avC_ z82V^TK!h&9hg$2SpPgNSXB z9H(5FuM3K52*c#$6e{rkN|k3Q$k=xfrVz*WIocqqbY2511=WBJfuV4~|1)eO{ttqP zl+`t$f${PwW(p=4Rddr2K$93j2vRf1j(q?fv6I$q+!#WEy@ddt6o3Yqx-o zPnvsRR%$@@?|#f?C#Tp7BZ)o0U&X*120gJIq!s+e8efd@@lK1B18$YX)$mRPeZ3HS{O=LAHaS3*#N^J}*URgsC5W z&)&9LOJii^jhK_pbst}UVSmpKrG#Wy!jU6($cIO;tB*b}RO}=~z@w0QFWTR2F!4Fl z*u$2v*fKW;MNksUR(lF`V}>GcJoj28B!L-PAG;OhNT(VwJHT52o|)Kb76x3sKMc5T zJ1Z(lN&)$nS|4hGGy+mQch@<=!KBmzuiS|ZSHh!%pe@D+)=0WwEknS?m%;6?C?A6k zGDvboIPPQE8UtQh_k{l?@V^!b-?Nf7lA$FaZGP^BxHKUjxYnapx(IUJ3dK^Ej?w16 zNdQiK07Uzey(;J%Xwb}82xKXRL#HE^Cw7Byqyb$s+(7IqXm0j93a%briNV83_>8>cdAtxBsGhW>po{?47*W(gK@v6!!U;T^i6!2d2939EZFc1lE0L&-D#RRPLua8l~Nk zK#{lugT=s3VtM!j09ylBNI9jp{0ULXCXL8C$B$v#!P#Q>WmN^FrcjU z&g*DkF8&}0+z zu@x4ZVOCKG@0Pb=5yWkv?p+E9DU6Zvu|gJpgGVXUJ3D)#!Ix2(f9G&)r8rLlp=(N* zgV3{-4`|jRd^d#VQO9BM9x_v;b3X!)W2oLfqcH$Wol4oqO492IvOxF{cdjVmG%UMm zHwOxAexXP8a*k9yRygxDHPnT**Dz6HDu7>s(XsCU|3+xpjhM6Q{z{tAKPvz3Ju-qX zOg#7QbMiQtxSIZ9H_`++luv3O(A2VYfR_18PKqFhQ(%Bv+ma(YK}5o`R)L@M(BS+A zh_U}tQh5f5DaN41qsqoXZ_UURBMBZ2s2R#+Bwx*KXV->I%U<%&Z}T8l4qJw3R0EuD zZjB-%>;eiSO2;AhK*<9~FpBIGGmOg$5+jPOwMGn61BP+qTU?P(v1mw!#s_jl0U)x( z#k|2_cLObyh!)5{CVa6^zNgt>$=|ni7z`dmXA>AaWKN~?HzH3G<(T%HDrf>j)j^~1 zbh{zAWD6|aD5_@N+I$t-@|6=UimX&>OfQ-h6m{85WqK#ja$!GnkG@2z|4$5HVX%{z zu2m%fs0u+}{i%^=Fj1F)%YjjwU_;&jzc{WsPOg~~1`C6g;dG;pZ5IVRx~!t>DLU8` z&~&5auEo?uwj-M&j4>{l=1>MC|0A~YlhVy}xK(ZyFu3geG9R}XfcSbefL78zd8Nm#= z;buFEqJw*YY~y)01R(YIujR0sCqN=9z8YUA?E@>alLqEjnbaUj7$#$04T=GL5S)iU zbEP3Lr*)12c?LEL6qKidn#M*~0N5#i!D<>M&qGlsXMLVF&p>Ebtb0(jgi0+FUTPZ)Q$kAdA2ZTQ0@h)pooDHtnx(vAIJ@PL;y3PDuC(U-mUH{eWdsCbkt zX@J6ABXTGppaCxMD7m3RW-8$Crc2`xVp8&ePKv^O-upjP)o%jWkQLLrhk`N)XaQ%C z@-rNEykztv9({?Z!xt*OAS|hH-wRPU?4qXD5R^J&1R;*kz(&%81M}~Sn%4k0Fy8MI zy$J?pX)@Z#^HXep6hecj#^SlL*Bphyl4j8tHNy@|Np{eWFPu*2#90Xgqr1uSAwc&L5 zm*OdUxfEV(D}H~s+r^^LAxm}}=+?iCV-4I25o%wWMTPHK6PJRhG?-*yxk@lA;guLa zIfWSro$J7vg>2U6H8%$!z2xQQZe7_hd3o{tXm7}B^8X`r9+qvU+ z!)GE-p&0+>^yNh54*RnQ{Y$lgv9UZIV35`(J?%lHT7P{7{N^ti(ksA>E0>XZ{+}kX zH5G3pmQD18cc!iwlqf)@NY0&-B^Qu6uQUc(Lre{byC_rP8Dw#gpX0NC7M-Y*$i9(( zxdYP}As3TekP8Lgo1mk3LIL9WJj%)P8hpC}uoK;MbJJ#8hdU#kP*;F_36&~0$khF* z`9^7`0h%_*)cu4J8ki)&ir{k>>-6H{c$YPv`~gEkb5*JZggy1vO}R^MjbcYQvdYA7)y&dk?%0p;&$k zw^G+;XpvGnijv=JT`x{}?N#a#4K!XymVxoop8XqRq&P2lm*)y*WS+|Vm!s77cE|YU zC!N`MOiCQ@PG_+bs$hsa${gq(N)S9U?+M_1_`gz}|7Op6t+b`!%kIVHQIb-KB9H_y zG42H7r0&Co==eUfvL(tbVBbl;9X1NiKDaV&;L3o4Jsl44-*ON}4$P*wWOIOm@+Iu; zPZ{58U}_%l>O1*X5QiZUf|tY%WJY?>eL06u^2#tV5Ds-^9Z*nOt9E#h+6lC3-j3j; z_#S7~fKxGF0K7*D`LjPPHvo|Xs7NLrS61wGDty4RMvOh z`{L@7EYU@q?ghCo&w!|sPi=6DNu^Hs0UBo7$}J9E5DQc77X?%HpJTJXgFQ4iNGdoS zqB$AkFFVlW%&l#eo_(0VYGiZ4uEP1H({{{FP;~K0r){043u0E zxN#8KG=`Q2GkE37@M0INbD3jk=$fkGu~Ha-P%aMT41K`b4~M)zuV<#!(%4K~hbktr zK;Oo5Ny4)@dtdQ6FnZC{Zt+)Bz?aba7BfBjTY6Pb)8BXi}%{x<{fuDHA9ZV~ZEM+2S{sY?e3of7~YIY5q12r*#e0NP8 z{G@mvAGY%=duY4S(%45o{ZNjAN9~hP@iAT}Yv&`r621Iq8A1Y5pQPLk$)8#+)kp%t z2zm7j%3Ux9;BlqCVx7aP1~7^rgs)cYdVszKP(ONStU+;Ud~B?xFnb}l=~mgb?{8Xi zJb&!qG>)69IP}ypdF8Sg2NE)DCNmf@*mA6lLOE0zAeY&*v>ER7kkZvcq7YM;)sW}K ztsJSMI*1qbrm_(3d>vZ`<;;;~Z2xh3<9-MbXWBb?uHlqAZ{m<*Y4IuX4hH`XcU z#+>Wf+HF3)@N4q@3W~DkTGgNHPq%={JLy|5{JE30#%W$YbgLJnNOdPqRFIWet{}Gl zsRH-KDyKiy5zl(gSjQi)mG^e@d{g4{&bZ9-@1{>Q(aFxvnO+@Bj?+4B` zo^#y*l3fX=lvQPK^W7k-=VQk4Q`wFe>^#d&I?T-HgHrBLcMui&hli)2mO_$w7{Vh6 zQgCN7ryw~AfVXnQ&OzmWlRT^52K;nMEeKy_IjSbL@NCP8gP-ymyeJw#HJ4`_JbL-q zDyr?zemg{bJ6@f#^y@`8z3+H$^-)6$6NPa}!`rfE&R5NdC)cS4By48~1ZGWw5_x7T z)ser$P6%>WfY4o_iPTsh-&tMyby&K@mcVms;!A|3Sp94}b`0t&7`xv%`u=>_J>*_A z{>yL3z<1nT!Jz6LapP>&;n~k6?vc*38{-UyoKu3E4R!xk`OzAfVp)-_9-x(45<27w zAi$A7e({pP}~#%r(IhlihQV*tBsmq#{kRR1zVgnoktFV7#4GJEX?`e|k zkt_1ipqgqM90<$J$F3qQ7Yc8bnJeqBLbd7jVH3`|b=_Qx^wa!`6#rI$JTT--)T6u_ zud>OD<)~GU8qokTqw?OW3@AUPygptSpnwcz3PWv6o|2dT_R5A;ta$Lk0+7Z->BgNfz z<`X56>jVVOv;p^MKehffe+&!zuPeX>QF-kvopiwNmiGn^LY(kiDTE17O`N6y)$el3 z&YVoBaQ8@Uk!2`RnE0JH%10#qzP36ouk#GOj&ykVId; z&Ae(6{+sG7aS&lrxi;z0teT%g2yD6BbRX>yY?{vHJrmnejaPx^%#K42f)+Cc0@bl1 z`6|cryB+tm{!RSX3PoSilV6UkEPfT6JdQkv;zvx#Za}$qLWE7b3&K%yhu<;NP95on zlJK3N*;g(ZgBkb>qErb`GXme|0)?a?3@UqD8oF8>y8;4U0gbT9WgI0aN!1wngHVm} zxi+QB6`4v7aBHzZ8BTd?I2pkKxyjc%zCr#S=EPucF#tSgE}S+X+&)t<&l_#!C!6LJ@BSWb~I|5$(@m6C4jAn6W4wIQ4hdf z2unXL1#q-TM_3dNzGYr4?>zGC|4&^H|C2o-&B6+K9&}MY8EG}H=j*{2LG^J`g@=;_ z*bFY(&$6lpM10%>iL`83;^|bV+1l*;4u|U%izJ)|Lo)A(OZP9+%~Q7=-4z8S*;U!?;^0 z$DwxmH%#*Zc6Uv|5TOS}F2NN_%JB0drL87}Lts9CicJY=0Lt?9?JOimmW%UBhd@mh z6nI1ez3FIgZe5Z_d-JF8ydDBvWsOr&yHG`PMPtn}hyh0<5L{XWnB+iSrCw_b56ak+ z$~zYq(?H$#tXhgc9R8o@r^pCQ4-30I$$KPDn8G#r2E!7&B?Oui4z6F2{(=|HuY5VY zczE{daBX(@Sq!N%CsP)P6-7oc$|N}j0{n;hZ-iH(TEcB)39Pe0aMmW^*kh_cR6*H9?tKIQ9(Cs?y%qJxP4l(8kmj?DE<0^E<(jFPWNeHkJ^><1Be_~ z7`Bf)8!~>RVWpj*>WJWUlT%+J@OH#;8mtjYtsPgcLy(>+Y}+}XSS{#h3q(XTjH3Xn z5`>Gjvy~Ru$ja|D(selh=c^Zn#P!YW)su%pY2_jbYlx}2PCXL-in zNA(6d{iR|+K=n`0<%>ZOhK2OAe|1$2!0-u4Cnc8M^5+Cya2xh2HE)xKztUA`%yGrR z_CE#U&;O;+o`l#(H-krY>E`4^B&CC@{-q8uXu%^7B>HO)G(u(z0^>o2)uU9ciDwjo zk5OgkzB3-6@CZ}Dg3r|z2VixraH>f7{c6pJi)IpD^y*xNJO4_8`y@CuQFl7Kp|uoX zUS+4Qc_UTu0%X1B;P43${{hgHmKK@{C<}BP#y)A~&IjzB0`-{T$j^n+iGo1ti;RNX z(2-R1q5lhAqY0hv>0Yfh|Nq@>psrHF9p^IXt5xtR5dlfgBEA-9xFY7SMDdq05z00r zlo+1{zK+?R-m!W$%=srz5S2un4f>76JV)~_n( zG_NA8d(ydkNH${nwY8vS?QqVC<44pkPs@ zQc`rg&L^kmQSI+?IjK9P({hQo;}`Rr63fc;eX54x>?w7jW1hk^<^}ZEpn|J-9|1`L z(2o^?E^y`o`|*$t0u_Q6W?Y9-;HX?^3mv$pfB8DfN>M~552%47tmdmR`)UFKif#W- znVFL^x+`UM%jPl>bUCSTEDM;wQdMCGRK$HO@pK2^q!<92zf(VO*GuMiJ{0(~27g3I zex=TNlpp)G9EOH?2ceH|Oi33cmMPr#4k26h$pKKh2D?eRFR`XZemQ^`xz!IB4VSM3 zA^!W9k^!~kcur(j)>N*Okjjb>Yfu7#icL*HRQl=#SIG!lfC^)^vQ0>}ESE_j$?z|2 zhh22x?Sj+P*X^qOGC#V9d}-+i~-t66^xm%4jXmn8O|W8 z^1)Yfk5&o?bpE(LqF@cn0`o#qlHeB1p+Rv03H=UmDML1GTkt^XbvBeX0D3UsAYsM! zsQ?lQH51AK_gT@co?Mt?h93Zr5e~4-f8(<`b?s0lRuk(#zp?;MAp3&mms&Vys9N>P zV{zm|Le@DU)cS`$T>DAMt@{K%BuzKuj2|E$a=Ex$4HR&gl<5=5O8&W5L%Kx~X{!Af zbd(B6-J~?WT;&74aqIfmCa^3#d{!snChp`m$nGCpc=Qyq`-eZv0~Us&z5N?bqCM;Y z)_Aqez$v8c0k17emV6qlVpk(NTT^|Dxj|G#e(MGx*M}HYgUG-7QWny;5XqBjal;^F z#A)0rp9X{Jz|y0x8X&5>^)?w{1B}3o7vI#yInVAIw0 zMu3fyzJ3PY06j`-AX8z_w?g)c3VM;gPlL^aIZ6$c0qPru?IGRj_|KC2J#kP$MoLzq z1xz3S!uppR0;44 zpzjuS;g1WyWeY7q6eUN{cXxmuQ`!ODu2b&$&Wb`_?ekDLV(?*TyHWy5 zNNmv)1;2Zxh5=m<#);-a&vu$TB%Ud-D^l6o!zD*0elD^@I(eB2A% z_-bOAQ-P>9WPO*j`UW6Tq)b011lBLg{WJ;WF9QnVNWO@F>09WyV;!}kIm&%Qq_Vu* z`@wU$6!N|vQWA)3mb(w4EUpkVE#B~jg;rs*C0>YEYZH;Xy<3arj(Dw<)Pg@voWV*J zgamRrH8)eCBmNZn6l8iJZ0ZOIssuy8cEp~QVB|TtFkNRza{`zIqHcM?MG~s=QAXPC zX$}BikjYmL-4^*NaXzukhd63o4frHAd+{KP-^=#?I1BliSw2pfc>gC35N;I3A7<=9 z!o9|I7J`Yh!Y*z_6C>_+?1LJuKcDrCLie5M&b!03zl*h3@V2YN4L%KB=M<-2IhWsL zr`}lnGn0$oy`(LcT2b+J0BE<}1p#D)V@SKHzuE@7(~7(-ca~uUVEmi?MdkKD#qxq( zfU27NLt2pX0&d%AHbH&04N8mO`Y|H^QiN?&O7QrDMCF(?&BOksW5C90b{S;yt&vyx zsNWuhK@io`#l)w;mWl$xSJwLKd0ol3@rQB~Z5p%fcGkmIa1CL3KBb)?Y-*?OLh334 zWY<{hjXj(QWPIi&6W})K{}(LIz(v1qxTe=q)EC>WO&kdC9=Dh|A${&!g>d}K*>uUm zU&p)qht9Rg`*pq&Sv5zR2>QR1%t%nT`!xqLp1 zk#8ZyFb*FNKoIR|=N_n?poW&5dJ2=FaCyyJD`G=2kvd*qVf3AL`r^s%QbKev6>l|_ zfPn4f)$D0{4jF@GnA>h)CaHRt`$Ev!l&s056}Oy#sx&Gm&!c;&5#VPV9jhOQePl&3 zx10$tDCq>q{3_GFF0@;%0A5z01t2AZC#}UOC{KWYS5Z}+1)B^jtT*f^I1A9?k?tD` zOV^zH!p#kaE<|cnY54x4>5j8-X&DR9_Brl^7x9Po29Cs^GekD2_N63Y?RAs)b-yA>fDcE$KR4iMMy3q%Vsh z86gK&_PO?rD^!4itOthQQ3vn>)|F6Flk1oUB>pfPUR)geDB`E@`HP9`IOk;G)u)S7 z*Xr^LP2Cr1yCcmP!~5fxzNQqXEzI&RWfl$-y=|q3+spfhb7%Ii+~Y1a0!CQBlRdfvI+PN8vkb%NJ3QfxzP7@Fq-Ns(}(bbS-qZ`dvU< zNCw=ezZNc!$yYrq>Q#(jFqc#(@Q zvZ(8PclI+6W6!R|;TKE3!ARsZUu%LNfJAQj;`xdnkfe_ohf?{hk89CoK~ztFAz~|h zaPDo?>RlZI3e=6Y-+DtK6=K}2+tR=|baVYpW9|?zt|@7u?2v-F@4(&2$avpo`iQIY z6Ut{w7H9qD`@3l!XRnVfxfdn3PrVv%?wJ45Nh}}B>Mc%|o9>H@xa!xN>md6u9qnyZ zxdf1is$HKx`-n`%nmY_0z}4@cWbOhh_X1Z>@9vRTX}~$=?1TC*c=tOc2u=(7rol1u zWB?0E_)5oF6V-sT{PDzBEY*rW72(wZTqCsIO(MQkpB)uPzA6f7UuYJno>o}c-hEu# zS2KUKGO7Exh=}0+pR-S z0?)fK95$3xap!<^$iH9${jNZZAaFS&&2p%hnbx`Y%*s{Zgd?!|`&{-w>DMUm_?zuX z2&GA%An7mKcT}QPwWl)aP1Swzy4JC1MWWwQrrc;BG1Pd5l;&CyL*W>Xq-f5LZK!g1r)|wKj;GRVp32%?!nq?R_Wj{lWI=`!Lnj^ z@wxo2ZE*Yob=ptf@cGGH7%-Zx;8irE+a{;8v?M=dOt($Ssj4n_Oaqm5i&Fo!A_rkB z4!!EI{M9@-5;9Uz&!_lf5^v)&7FSfo79{b@N9F){j)774?t;XltX<@T=?;BVYb{Zt zE1|~pP!;ShD)>_f^G0905C;QU6S#aqDF#66E(s-H1aFXXqzR+_&&(IAqw3@x&%b55 zPJddC;`sVw`-FP@=p?N?kVkls(qYl%;bGsV*kPkunf`j=l2g4x zt9QjocMk@rkPLvUlo1kVhb%Z0x0dAUWxLa(Q~fFGmKPyZ1^) z^xxRVKFp0wfPX2DFiszToUXHH`SYieKRwZnnbq4vYpxnae*ZKSOjR;!!X;Se)5l(Q zRz$93A*It}@N{2ST@_Q@+?Y?we2#^OZ+=V57oX1f`0wD_EGHBR`5=Qmx7?x5h4N4F zzJKYOjN{r0g3}Q0>Q>+bqd&5Tf+j%z$tsgsbimiAOOm zirDmn3lQQLjDVez@#hrp6H{u`ar+tu0c+8Ho!g2 z$h*sntm8xA%pO7wl{-TAOim_Co_Q`K)insv71 z`YgCwHSL;5{7XfF(N+#|Xk-J_58)NSMvx&vhaMUvz5o!2K6HzAkCi%CZJN8}9&LRu zNGYB|Ai8-y{yL+)U>B+;WUiSyWy z>*%2SqM3jr|4J_6_dPEl4GOu+oMe|viR@wg=a*wqnK0?9SoH{0=3m)7xI)_jIDmmM zai8F}7a#!%-L5ro2V)r64}imA_)^vs$IRO|OKEanABl8M4Rt6&xEE(JG_5Db9ND9+8X%!-jEn=Y69YjfRzk`e5Fb>uGS09yZ1saX#E2$l+8iJ1 zr!VI|kHss_`8j&I4|8#WIlaBHR?%*lUSfp(Zx(==-J@n+`6>&tYdE;Nrd;0@0$=+X zhxNWcWTOP%%m8-$lv*%32H#iek_CT#`Qs-4SuJ%uUozA$FNeR&f#phF_9*&z!@pV= z2Cd;@ywgPM^D(ndVxx1s7ku>O^>`fX_7|7!3`lngFqA+IW?NhXm^%1MOhRy!5ZnaN zfgMdK+X^)y4scjsUb|4I3aHhCQTXD=vbp~Axv78o$WaqMfS)aG3vF>vkqgj2%hTtkktW{*GrcbKMx_DzV97Nf4GN>BEGvWKIdJcZ zbg6xrnW#n}tD*)nDgV}C2baBs!53jLD2iLg{MLLbngHjA;;J3EVlM$D`N`d*FZ_s< zD_xON5;FcAZbbgeH=3@?JV+!YmZe&VUPY(!zc75unj@zGNXMc>GZA8!o2yz;uLh z_%P3avryprV?8D9Ek~Opa>41QK&m~2*dWH!)ksdgge24l?W_S z`2`5*QF7$~1KAV@I3740zW%bXNYNqX0D%Ci+~V%{uiDJ#2Oef~`%L+D5J#O}sSnfT zcFuE^8y24nEx}_?#oq>hNEV0~lrp&tCpmlrhE0>-4lb2a-h(Od9jzN+?WsF(c%4Hc zH=bXl@|g0?Yy{R85MK|ir)8vPTbN)ep}isX z&GW18be?r<|2@Nvjvj4%7pazrfF5h1wl5d=!Ivf|fCL%|b+{ShtHckrXyIcxED@ae?7t*&x6N~63hAEyGlJk|2@eZblv3_jMLa4j*`Le zfddl#XV6WyMrN!;szB6|hjPG#7k=e`X{7WSNIf;J4d36W5WwNFj*bTK{d{ueb)VA_wl{^OXs}LYh17E zx?b0KUe~*DE|K}}f$+K4=AZUaZLsSd%P7`J&Y;A{XoH*dkmtE4cfSpYPv9IaOfSQNfG&Fqgo_o*E zfjHFGLhf=+5~MU?CAn_p=z__I_1V$+L0FlE3_jhR%hL|PF8vY7x?w-jMcUpfZBfV8 zRPDw2kkT5_F_7#8|BW#3IhZ%bv<=#;2*XxBwJ1A6M4b)>&=4_V1{KW?s>#@(YC;x= z&3}5}+gZo~*YZ8T36~N1T^7S0Wh9h#a43Caa_Ubh7}5`kmsw|;b~0p|j+q?q59zXS zu$vZ@r#8qRq3vyJ&OOuOvNT3kG4D;*t^jRJTpz+ukC`SV0-p6AYZ7h2i7n^ zl?jc&9)l)2TX-)+5Xc^aL#3E&`bP~W&Y0hV9IF@br+2*_st-604+q&(>TAL1)#^M2 z8%U_44UC}iVm<8SY?duMbU@t$k9Q8HX+kC;8fFQ0Rs`p!6Ba*sF`2GC4qI~l7~ntF z&GU57eK_AVWiT$n;aP%%X$2$z;loE4lY?I=;jnD2_XlR_sP^CVxD2654Gy)s< z@2(3RTo=-j`G;XlDHAx=w!81;VE`vell3`h&ce^2>Yp8Y3>d;GyfE>31Dm8Fp~@!* z%`nLTE0cO+0!X~)-)KgDiVf6thddfleo2WC~@7Q%UJKz%vbLe5f--G$)H8kIAKJB;UE{T_;<<_vI=8>^ zyhz=M;6K7r4Qv z?(p$ctn{(O;CuyX_xEEDp|njLcxT2T8UnSshd`(kUD_d}Yy?-=%O>t7h?t`9$7efy z;rfx$`CLic%PyL{HP9@3|E;zTdC~>&PkwWaC9zRU(^M_Exn%x6*XL4G)LKpbd^|{? z%9;d%P~FcGG*Lcmhzpq6(!7l?XkQ^z9zxCnhqb;ayZdt zUJ0vy`hE-AfboWQldS!t_DzgSlEt}f%X4?j-4t&SQvKZP5b-9pbk1uGq3qM6Mm?S7AG9CUgX%-Xe*0rBg4q)&@HZQyr> z@VkY%k2P=yodgx+C(#B~y7uS}YE{XDJBS(9=j33-3U?6bn@0xPV_wkl8fN%1s2iq< zj3(I|30UdQxq6j)7XmccjrQ(8MHr@2Ypq#%y8KgRVz62wXM+OXXbBZ2S%9n%6 zV_?mM520Ec#_T(ex(oThD)=fx-hKjYlpd$`ha1`K@+4NdtO`S<9!1@m zrTAdQb?GPe0qI-dO4Hf19W{qzVTUJT+n%GtKMF>HYH=?r8Go?@4nOJ8W)!5_08fza z9slcL-Y3_=5_r=!{sOFR2X!z&TanrFajki<>O~k^c;6b@(FUeWOG907Nh)Dm{+Dh; zTRRecxN>;**v2CUN&w#{hCklgK-*%*?jzi_{s=b5d(L-_W=&+J@((p}QM(o!CtFR!*Si=DD5d-=t zA3mCzi%zU0p=x^^-YeopcKmIv`B2}ctXd)QyJOZe+A6dr^fFJqCzCUXEyU7DOct7owjiwBx9jn{ zKR&Te=1Ja5xvl52^g2=>_%e8wA8mgaj;fKC581lkI|yr+X<&t_w7b}1_9re9s(?>Z zH%`OJd+BWbWE33BQ%zgT;9H$N@B@>xV(aP!ZvUH}*I{WX zQos_w%dk$sWr`hxE7$P0xB5;#!)@)CzgzHkqvKuyTf}#?9zAUMNRGn<=y?HIbS4QV zApJh@oj_tbR1Y76Q+t;ZN1JufP+{BAvQoJHN7eHADH-0j?|F)p2|C&zNduWqX?F(~ zO-4>bTvJUuw6Tpt@>b7g4f2LmSdvXwA;_CF$lEX(|caCoxrh# zTxml-+K;2I&GQfjdUz*W{X87m&L{5h1kZ68@L8QY(g9IGtn<9WsFmSyq+BQ*mAnz% zeD%Gy@fnt9#HVd$u*#U8gD~sQ!@DlS_P<|U2ooIsgZ!|$*!j;S%{eYmZhhgks z1%B{ANTQK!F;@5+Y~wqd<9NuJj_(M?=M-cIMFs8sR$%cDl%W8QIM|X-ets9s^ZKSB9F50BoLwakqr1S;&Golw20mz_FAim8qv; zrSl6eeNCeNI4(-3gY}1RZmM%5(;Ulp1}obO_<}v`GEhMiZmz+b_%R3_?gY|+NtnjJ z8C>B#WutCnm(urz>?fgmZ4~A69mjn)eDQstEcyMs?@tUS3!*Q$2kwf(cgMQ}9dC+E zMl4oZY0NMhw3WU_o=7}aT1C_hO3{BY{5x&fK9NZ<^`x}>VWkf!`8Ndj#D&^=NYnpd zN{YYdUPK+ zW+c7{VL}6@wKR#G0=n9F0SBs*M4zOl1c7Y*TOj`y$iD^hZ-HFbDu3UQKWo>&Z^$3+ zvj1P-kP0MJWZIV9vOc$ITz8RP(Z~sCHuN40Kq)!W-|<76Ro4o#t2S8T2d`gkZs6=! zN6k48kqp!{)YKhH*?U6gwks_ME3@$pTQItE+Z{zAmQ4jWJ>TL>=uVUik#byTS35M5 zoq|}@+wRC{4lv5pgxnTLsrDPXZHH@IPA8C2<({Uk8f{IHq4RDsT{ZR26Nsx)Lch;L z8ReW5UxT`HQvPEe%EaCp+Jk{1cKf?Ll-9T^@~Y5#g1`D*KSr*RtuBpi}>n8 za6?KFW|Mm#jBqovK+54g!3az&2g~G(GlnG3R~ya%;bVd!+-IoPvJ6y;1z-4UOqvfo zb=*r+21`nYmvu2}%A40cYS*{0r0vE*w;^8~3QNMPnU+F1EJV>BDWv(sOJZ1II+0eJ z5b&{)$$E+*3J{ng=;T^IwhKYk37DP->yPMx&3_KQznJM<5={y;`eZMIW2r4{11MZS z3{jqEhw8*q;pw;a6xPTBR)@z;kpUTx(QJfAIJFk&0rRr@PaRV6zuhWGF9Y`Rlz(*H-16@rfM{x$9w<^ zI#9rr+s|%A(DsKgr9O$-Rd~W5d>=6FdCF_g`~*ADs7JDHoH2P%r{ni9IlAs2>>)m5fAz$$#rh1o!U zYu!PP@fikhSRs6h0lZ;TDg7$5*6olNS&^GP)*dwW=|Dz z2LK8dl!5&Y{yac&BAlTUy zBSo&OIGsu???qNa8eNt8#B(^7&aOBV|M|xmT*ctrzl_FfMymyk1c3^`M!(Y~e-IZ%4Lg1idrnTNnBKd-xZ2G$sDcJs^z zWni;chQcU^I0lT7Ze|K?Wxpzw&#LfeIufNM+lTQK3~yVWvNazx-$d5u`WK+hBh(;Ls@<@;aD1~77s9D6c2+|uc@*}m1M!;(X z!uPq2V^fZ_e7h$eH$}RUUCpo@2myh#r`-ATxbTT9DUgd&sAC^47X}>pvk5rxH+i@-+Z2&1f?n(~kBNpMaN%{|yx3W>ma%u1>XcE{2*m_^@sUI&Su;#)piXctKW?IMq( z{Qml^CTu{v3+Ttc2ctkNydVvSJv4Yfa2wzKY+>1_bNvx3BBQUTFRrB(?OJjqSM}DC z|KoOf>K;$D=4N{P|Az~B;%^hUX)pa;i*uIwqNujOUM>VP(M4fOMJYbi_44O~j|NS| zbNYhfHs}S7)7ijx5*~r>;aF+!7G!i4lfaL6VCm6~835!^bMv*Mmv+A&qL`6R$(74D zx@0DEZ)w!gbMi5sN%(&}AzirL?fD*0i9K4gFIYYEI)?;Gd;f>zR6f5niZ7xMpX+G8 z>%3|`VgANKUV}#oVxgX*{J@ZFNy=|r^PS3cWdqs2K;LLZ!VbKuL}Cdh3yB)Ckmx&Z z!x6*+$rZL$_aX+m1PpX~^ladEzWeFC%9V@5SuQl;lL|W&^g|cQUevl^7c5@wt*;ei zWX|qU%uFzgSm~e4j?~_kWIrqZrd^v5no=>IXL202`ajbfoI{S?L?gS=25qJ!i;Va= z;^UIkUIvwJ=TXR#LCPZ(nLHncgLkgFHA_x7^Q#=PibI_NlqigRF zZ1DUEVU&bA4vH=4cZVCY5Uu_A==JAN%kkQ!eeQwx;wLPcCIeKM(+uBXgCJd-@T&j2 zB*)Lwwm}flu(s_yNjd1>hl*3E&t)GN>i1*lP%f}o2&T7ShwkWir1k=-EoQC-C?hZk z=1m51vVyYo601u@*I7 z`}u(RK<(gud&Nefg!K5G&*-UYB(r3L0q9~Qh57&Vj*#M%$qvOz-<7+X^s+CRey`o9 z=|Hj3R(a8bVmFHNsC~EbsXnfJ@||A&c}^-|W46M({yWbf9H zd<6)+s#*`JwnG;`Bbe`Hr(_cZ zgNxvNSH&|={t1?7c%^~UQG2|t_JARch2{36ayXWfXf{+>^pERyO@PCHZJPYy5D@H< z)DwTe*8K!)dI=pRzd97uf=ElgBfydi3&P53z(jv;3Yp$t9KlflN~L7`7?7vKL4Aca zK(Bjj$tRmL=#oTkrhtDUvIO3vtHZL&$R8_xu~Xv)H=sg_?d zz(=j;M1Ed-Dk!42ZS>aGr$Jsg!ZPKdKLbLcWLOSE9U=$ttL9L;GoVnA?(otKG5vrb z4-anx9$lhmO{H5Oc=4o=!!>VtnMFCrifb)r~D{lvj|2kH3?f z;w^~UcTCTr5&sghbnx+T$L#J_Q81ip@iz(tYuTjEv{r?Lss{3hmPOqroAUfLbWRr) zYJK8oMj?}clF%5%A%BrItEYH_s|+*4n_07zo_+A1y>;b(OBQ8MQIeJd zw*~p#-mo&x>*^?Zvo4-`cG*~I4>`up$JmnC`%B=49Ze}69y?*-Y6+SD=d8CLS;mh+ z7WWkC_j%L@<5J)V(Y#a9X`16GX^pLXt-B~_c5mKE z94s6GJir27A|2oESJ_Ai>`3N1oGVf+rUT4}a^)D)5Y2Kf(!!umvYnF-!T1YYFevg~g_ zplzTsPIZI^oZkVEONW3eq;R|2Py#i^%$JF%a%Np@k0aWzTkR(B`ZjVUm@w8fpq*UWy+cGqU($_CR{`F^CY1?f zY0dmt4pat-WO;I?4Lo!N0ld+AB?K@F=tg=NY{UWa7zm0y(v&rUw{#FQH;06!HQ!;&}3WrF+Z0Uv|m4d+1<1%vsU zT-ph1BZE)1(AyP$#_gh5#WH+7ML2s8{CU9jV%t};pnb~^$g(dFn_+hr8;`=%@f$=F zy0~`I2?Zv!?{+A-5}1juXmWp=?8=(>(g9BO(nTs2f1x&HxC1K`zZz5V*q@zevl5-6 z=j}ZtFqViAY{!sPeD=ag%A0XnX6wSEKe*+;b_tXq#t=&$&+cC5>V9xdp zwa(J*9BRq3KmpC+%z{5Pvm(4IKogi{@N(7|l;*EH%`Gz{w0uE@Mpnnc$Z3B3l9Jtx zOonOK8&=8i8G)T^G0OFUfvx}igK@jigxgk=k5|;;@C?H`+~Av7h`}j5s49U!zlR9{ z9$eb!)h+6seD5+dFzrFulJ27&uYeO2x^bvO3oQBjketa>(W%0!(ojXv$Wy9 z0sHA}qV6!@_KeZ4MzU96R|tW0zVl}E6_k993|7Xmys|l{{SYhtg){xT#eO%56#!JQ zmYinRrGEj9fD_rD&?5zQ1^!g~yms>JV5cedResTdplExpxgrAQb;SP_o&3eJ4Z#?} zI!|*Y9x=p4nYKd@ZZ?qSmo5DKJ6LdsK$^L%+qD2*fP#b>zH&ucfC&ycBM2x1E6#Jd zEYj{T+hgqhBJuaDMuzGW6El+rSoI}!gd^Ai>$~t)AH8Zw4cV4HZ71Ldv2fccJUg2A zJE)cnt9B~5%8dK|cn{k;oOK*^PXBJuZ#&RmL*x*W62JoZ3Cc_h%TBUtlYnJ|vjLfT zxh#EHw!<0W5G04^GLOK$tsZ0ktAW2^;LY^FZB67+UqiZs%KeJERRaUi@LubFSQ5Ou z1gEnh?qm)TcZ`;iX|lCFn|@ z0~=T`j$lEqmw%pMSQ{x-bpP(zqi};}vF7xFPBnh!wq_3exMq- zxW~RCjST^MfxV>l1nK{^`j5h+WRwgy>dK=8_Zg<7`x(R|exJVm>zFBdHHyg2iNDL) z0I=0OV>7TKkbQiTZp`&J*HB=u+v7TI7__qT2QD^_Z zs6bNeEy>%>x|!?a`Kn7u+IFZ~e7>v|IrdB)_3cvz2R;64vTSp8SfsbIrhJyM8A`+Z zZwnHY7nKRDyS>HLgyc#>$G|>nnjlC{l{6laL&1nM;2|+w<3~M8qajhn!+(AU^5v2F zpvWp`dIIIgt`va%x#nm-N-9QPc;0wAdnl%E5jRaeF(hyq+M(XyNWqVbdiaE7-IZ-N zskHGEKq!1A(vRTIjHZqRo0mlGc{9L~8J$aI4Hp%Qx~sU2`i?6}$Wb=PJGTdg9Dkkq zL?NDZi%JOb?0_|e8BRYA*$Ht!RzFr!z|BW$f_IFp>Ch*UV6yXYz(sWRy*f(xSHXpP z)a(;EP@x*GNS%O1s1dzWjA~S6m+Vl8H4Rx#)|(Q*yKqI>vw*oNRLY|dDSsZ>6`_v+ zMvbKQxw6j?cO)6!*?0c}T-L8Wj-b5y0T|@~3o4diK>@M*!OwTXJP*N`-0j}C5B@v_ zHF-4W4)~)O!fBxKF4rma{>c3(+WK-^C5U+OZulYZ5k|+e;9yeymvuI;;LRiLu!hd@ zqRt&GG$_`%EPfdRke#rnK^x*&_;0S+u%*6n`8`cgz4lzJpC1B49lxQM&#Xh6cOjZ1 z3$OA988R67;m?n;oQNqNz=987KaMQ?M2LQ+zqtiaXXxwR9P@r1}>WPKb}NIeAA)3pQ#$Yx%ento9cHEdX+T%Mh*cV|@c0Hs8ft zK;eJAOcb|-ijZGAFl(NG@bGB}ADPx#dF|i}@{PQ@%ARuVB)Rd&88gCtn)^yRWlyc` zea^J8=xsx`E`I21WGA(_Lpb+H-K^DHpUYs(h0(5wo$cHqn#E*STFYejarPEr*CTKu zFK&6iK%5Fj3Tmk zi?c2_NOoCeAD4FDcW;j(EIeq#3%0*D{T&$;P~LWbP$|WK1>~uVE*w;5^`s76Z8fjw z2;Vkt*RH$@Bawk{HBnQe*SI=u+@*S z)vH;}1KS~jcQ)U(5|ECLK!{QGTke3TLtP~7Ha>;O==yV6jRUBLrOXz$!nB&6Qo1$1 zl+6tD-mOo|zCN#%SV-*e-N9^Uo&7OfWsevu1o_{E1;YgVzf=9Pfap(T02T$t0cp9I+7EyLtyO2_VgoH;?X^Kkz1f(&O%NgVTv zhS2zDGb%IZv9wYXQ~R_MO9aJOcj%g3XN2CQ^<=9yfyFO_4HgBcgFuO zSFrjCUqnSMKGR~}BE95UAvth|RCK;ZSdzkXueFn1OTAmg~DUG_>B zQCp}to0Y4j1hq-nORUVgw?ljWC2$9ExAwp67C;-4^@G4~x)89VO z{J?Jr)3Pu08a)2Jl335N9oqxoO2d=Pw+GWoqd$a{4ox^+RZYEKW=_X#KPy_gbaBaQ zzUMtR1;#dVWGe{a?l0NAC>Y7c68z^||9P{9XdKKC!_3%B1wJ?gJk2oirrK7y^xUh*lBtJD%x~5 z-;e$W_ja8ie}bm|32#nUp)C} zWxP;FWevjjKhOj29;PkXYzYOUUsCW*w#<2Z{yppj+ScN*WZ};tlLKeEyJZH*AN!Yv z;zNmIo|GX7IIcgbb@(JH)iI+_m)ij-alXP*8MNcqJ|#bM@+^FbFxKDWt{cFtMus%7 zJ^ME20WGhNuVR7ohrE zXZR)R`Btn1zlg3hg66+Je2j8R(Py5Go7ZdT%-%X*X@L#8bTTZyRYSLG%erR*sD(h z*bRB&LM{4X^zt}6bSwQl{0A^w*4pe?!&Ae9^s+GG0PN?VHRK6-167D@R&n}1Yj>me zT0XoKrHOyIoY&?`N6lq6r=2Fh00Cp3Z6VTVi6|eEb#k&Jt$F~SCC~1h#k<#XCvibK{7Z5 zHq|-0w5$mYI-1|jWPU3@DBnDq;*h7H(WCL&RCkbK&qMw%EHJ!2oB7i)70(aqnI5xy zl7jT;QPANxSKbZrncDO=trp6vc`k}Augl72cJD`>OLe0EgfOfOJo|Xf0v$pNkV8j( z2YSmA%V8i#Q10_x@cAvkJf|c+kO3;ucmw5*h6QOdqVje#6TPp#yA;WWGRfSD+=f@? ztEWex9AwZFk)DVB`|u2db0s9aA>ox;9PDCf>!;a#my)ZKMnF5&y1Z$v4uIqg>0FD6 zoTOn2G#_Jfu33LP5#;CAN+N>aeMZc_;|qdN>BFjc)G_f8*3m(*pD3%`DL`#)T7;C? zdLJ58TMO1G%y=JsK9F|0E9M(4+8rA&E$Nqf&uRmB&jd?Z0P0|1Ur{AGKXr2{E z%Y~FejPP!lZQC_zy*+}an^kVLzV1n_6BV6E?;ku_x+LqPeY3-3HMeLPu68dMIl=6% zj5pAKN-WeeZV#P?-8og%Dyv*Dzvi;%y+rja59ZaGX83=$O^bymzbw-T!bQDhF<)3L zy4w`5a!t=%?b5fR>&Lx4UX2vGmBbbUfF z(!DagR;D5c#S5>JAGI+ZnRU3fqAhUrYW*wacxZn`SvzLAF>|hhsQ1TIf7gzIa=F`_ z&aJt#z0hcnZ8W`qYwu!n!N6m@b#uLCeIY6sm%7zl|0FcVn-HwMI*^}o`*WpvSV0o6 zkT&4b-_ z_Dp$+kYPpj~+5fuCdb@4b`mbIgv_@-D0G zlem+9j7uF)n3qz_Crc-?dUDPsMPa7myFFEkhMapTwO6_(7aj4d!Ye86OuM(bw7gl+ zl;!TRf!73)kX+9V;L1KMERQK9+E!m~S)98+%jKNlfY;WEHz9 zt(rJ$aKQcCGBfb;DVCH*``j_(q1*`jYK!oB55isBLg>r4O2}2T>rIuZ79X`7dVUx5 z?#qKIjRs66E?k;Hz>%7tEomd{U(Qr20XXEI)RyLY>u$<5Ht4kmPy6NuSxwojsb?~s z`~&%c4n@yIqbnDjOdn2s?ddh8hD30~JH)YHHe> z2=43~dtCdH_vR0MYIu{eRCIbmcS77E1dnGD`b5FKcUiyDQ^GtV$CIo(tJ@{d{M~`^ zqMlSj*y^*%$0|j~N3B1^-^=S*z0$iiYo)@xsndp{oq#DF`9SPFL+Kla!9UIzdE?^e zbf)Dwv(~?gg4oT2e=>|6M;PT6LXsolD$(Xk1^vMJ>n{50Mcy(m77=7^=~k_nNU#HsJNHobZmUC~^@WWfZHZ?A22@3kVw zZu?Q*GRYUrQ;mThOP$lcy&lc&#wEO21s^_a!uRHEV$BKQa!Gr)!aOxol9Qk9TUhpi zD6!nTOi9*U0jh7J)H1@wP^jm3+Yt5r*#S%_j0{a5c{OF}G(eef=g-$q7RxX)CS_JA zzxka|q@F$0ttq)YV2k&p#`JmjhFW5FZ5_uXefe}?)fq?CTu+gIrolu@ zwt1micQv|jDva5qbg6lwj2lN!#gH8k&ixtSu&I>(3)PW{wNaGuz~bH($`UFT;aqQ> z0C(Xfb##rFD#Jmg{87oF7)Cn=e}-Xi=g=IGb&qIjzx_G2jKOij(Qx9DpY!Tuc3(NB;Iq5?RD{U1F(s2N zO7^7f93&n%JE1+k5Y*KY{InjWIWhp$XpT6V?ol~!!I?*NtsU*0b@iM0AZbujr1@}Z-k|815W!_> zl-Vyte_l1rc!+srMx~%Tj1beSQRc(EJV@tm&1^fv;o+C}=UK~X$kMeliOUBT1SH)K zXVQ!$(lI-#Xr$`7E-qklQ*1}_#B-a8QIw?>nzGbv9biZP?ka*GDocuzfjh*JQ`7wE zO0}+)&qAXycV512?Iy`M+sR861XD-9la946b$dMJzfHD(gSEu;ZbFV#>R8=Khd@$P zRUw^_JnNS?O3_-Tt-yQ3J_Rqmp>wUcL~T&`V`taGf5i(tCdJ0(-}`Z5knR8MrY&91#x zU>!)*JBvE4j&}}>Zz4c>R?$i^F>Hd^(pASTWMo%;U2wpq1>>Ts(~(+6dVU*=-AupV z9tr1ePjPFSBfBNcKl`XGH|LQ_vdnr)YVp{8Dvem~>sgJFB+Qa}?ke%$_U#;@+DQF+HG@mFw zpv|+zj8p&kcmZ~FNv?Hz;kvOS3LTql_7~6ci62Vf#u%7uO%Of&dK>y{A|?ut8O~L8 zhYE_BcKC=i*Y^)_FmZG-GuidjmB##aOh($Wbm~mD;arxSnh!Z0!d{Wwtzw18T$p!84gk0PYa)E&8az8P;4(!rV<@Y0SPpHXHM}X zO1*B%H6_e@u~iq&eZRtIwcM5lnBz=xly`rn7JgTSuL~SxT&T+BH1j75h^^*BiDLI+ zm){$R{n;WL`#I5Fv3Gn)(q-PsqF=2!-ywTJvg3%@tOYf7bL$90Dc;m}f=Dh{SYo|M z%(Iz)7gM-%(I}JVN&>lH!TZ@a6cv(e8WU}fTm4pMu6C0*k0(1$hja>Q{1oo)BiMa5nw6#6TjN_J8g`dn(@ktVTRZvM2>M$7z`q%!j&rc1f9yvbP44r3vkdgGy9 zk^_QTc75-A-I*6}*)@5G?;W>^Y2OlV%_;8o2?~_olV2E7bC?m+oKwra102qzsk?On zxfo8v)ybYv(s*!rlC@}aw=kZywI#G|=zKCX=0h%zg8MEXyga;N9Kh6A>T2+03eQcr8eb}eZ* z%!p(q8V|jmAeERtJKGcoZ57s-aN{qS&>?l51q%WTY8b86Zu#_OgYK&K(H>rj9t9(l zt2+-6mS5Wt(o^~uWDgO^MkXJf^krN&Z>S2-;yJ}rnPxOi+dFF}syFG!;W)dItc+SF zY^h({L*99Or1w?c7}%u2x}|cac3}#8ZQGrx>9-V>%+75lTJ*TM|8{c9Z_vQ$3uP~R z-uU*5&K0t#budrvDP(f*VR9Hu?M-o?qxQp8ce~&+gi_``2`7C;B*Z?y^YoOF94+jP zt96!}FRH~HcUrQX*ut^2&tpN`Q*t({cNC1O!o>$!)tD9&WSs9yX~bmv>}q*yznDQ> zS)I(Zb+$`sJg`{e*5`Q<E^i9V$Ord~NCPvOdjkq{>X64b?W$kW>rHbB@%tWW`5^KU{C{{i>-#|*|>4zEM z=*^d07XFqKv4W$qFs928PsDWQ%S(zmK>_GqCSZlNRtyg)su$IAYYdny8zaUszPD#6 zFK%9u91*e`40hBqnWTSLK*(`M{W;G^_*8!wTJK#LdpkUQF2m{?x=a#R6iRyK?a$4Z zCD&3iJGFb4GUZJO2!OT{2~Y_U0mtbScGQ4L&WTHtpq5*WXTSGgdgfsA7G30@>3nRi zk0ev>Lb_^YsWLM1#o= zXw+WgR)HVaa2nEc$7@^ly|<*MDvDURXGRsiRQwe4dht-QOGS?hQ7~$Y-?O|rVxnR} z9s^4x>6<3$=Lx;mN&VbhDvzatOu`_2mwr}UBbMR{&-c!Oz2&3XaHLo zES+04@$HOZK4v$$uflXXdlOtzH15ThiM;|vh0!$)tVQ@p_i4PJ%VPXO6_qL71O>9` z&18wp^;J%svo4BrPFnhm6>zg*U56#{FZ2bJ+__fcy}h>0^=kz{XD=YuC2i)bX^0#}*+#PGX_6}o zFG>`_zP{H1(Mwu%6wyghtSi<11kAkUguv3?j=c6&zX{r+pljn8)`D4gcjMK*w1K`< zZs)PlaN{8*EuZ0oQaYhh&uYbj7d3KYV;&d})uF&mg>@IY#aVbpjE5G^D1UX z(ZH#Ioj4i;H~D~i(^cKLHQrm8&F*A`sNDEW+eN5-x+C?>J7mX22SN|?xd9h*%;kW- ztb03M@8xvot5jPSmsD(76%@f6F|Ty!d{C6QJus+Px>P3FHj3OVukKy^MB{PyH#s`e zmu#opHIZ$CXa30HMqOt1iHYutsyS8^8Bc$#AI3{Kj0oY8WPEMk(qOYV^YY6su+aJj zOTD|kYWceN;s_N-PjS@RG=w8=K3*-WpyIiE(JmfB%gb(U!T|M_B;#6PLNZ1#p(exX zQmJk2!uuZPapuYUF@ymZ9qV)f&>Q>5QTz-sDrHPAe#qh@x7bv2Zw5DJWwfLLd=fT$ z^p4 zZ^0?PnBz4)bG?%Q$=Xy(BzJ}KO$0Y~ycXP4fi_T8=1)?oAuG7>tRhsolRvoluXHlgOAw=NLj~+c6YMeixprSDDnfbemRTid3!HgluU6BEvuxKevBYN|mzcVj<%f+o*kHNR~^p*|En}hC1m()~(iK zL6LB7QyVWM7tX!tMvCe?tm@m088@38K=Ym0(2yIj?z(ctR))vOG#{y%?q9vFYN27_ zthJo+4*7gIpB%!o&k}pi#af?-YO4bqVCgV1_^*_&>eftU6%?dkD8pBk861oxf*<8o z8yvCnLU3z1M@@fHl<1f1nQ|MRZ9YnPCiAQ|N+#K|9jx-UOKZk*16{Jow!u4ydGPmH z?YU!bkB2Qy{0`^)UrZPGh%6O;2fgZ0Vs)5OZ zZt74|TL|{2HuZQT=i@bv(C{tLWCZP8?#HImif+QIk`}oT0nbg0P8SA=n{KsCblGlp)UeK)e7?%? zeb)S1L5$0cWp8$15&!aZ;%Y?7gytO6=)w~N7fvwJo3~6jR!v=-9ajFzW=A(j&ew6j zIpAm4^%&m+u_Q2fm1`Q10xx}#jIqt0n)U#k2n=-+A)w+$1}n+^4(FW9Q@&{)gZEZZl~L!Tp3qIlpnSysFXhqkp7S z6|Q)yYr?--8%ch><3()9@oNd+aLC8xT;9jTZkn*%SwDuR_%dDUF8@V!XYT?1J(BHV z9fH1Mqs8-Gzb~CBC7bPY_mhyTwUpG#79yJ++a`W-#6QWv7+>!#q0Z1*vuArm%s`!_ z3C4=gXt}8)U#H(!{=+kBVe@XQfE+7ndg!jj?o}+v8+dV1%_@|uN<*aCB6pB~pm5S* z3$rM%2vhUnxz4=PjmcTEPMJ5ZR?kS;eeXoOc)cy0C< zxQIU%STk31pgG5?K3t>A{zl%Jy267??S95iZ$cxT6NynQ6`Z2v$#{bmZ7I!MZB zLhQUFcbHYypctVDmXV|$P`jbuYQH{!ieFIpd*)bO&*YCFeQ%%RThL{HP)z`S^+L+X zatKD>879lRRqq*o;I>bFm%aa-{gV+pt!)#}1&H_!8>m(-Px7NHwuAiOlMu|&d6jDW z8&y*7pO|Gt!as7akjAkxpnqZ{@~4*8z{Gm+ZAQ^4tqCfcd;wApf=={52OY#9g767i^R)nf`@Hb2>dq-u_D8mzZL>p` zk8?&KAV@D>JGeKjErU&|_G5KacPKGg2>-&EvPsd(V34-pxaCdDp%)@^6g08q8@k_W zhzuFjS>z_bDU}bsXtpCpp630M_2l@(xyLq2$(O|EjB*@9!TPT&ct4TlP*-+eBys;_u4-9aqZ z5XmvUDna?M;Xx3T?0X$nU>c}bU%L88HBfJKwkGn0cvu_Uf%_HeHAIBNt3>@?SPp&4 ze_d^9nR5oQ-cmkWm0fsx_mvzH5(cHDrNk@{!6V8n4-R4^XF1}RGTPDy&6VwV0 z7wmttYm@hlql>rB@!eFFycxavN7vdL0az3|=C6LVSZpYhZ78`e+LV|^s%{ZMs=o8+ zp#u*RYOTe%qu4zW?qwf|*B;5~FJG>=dNrl$e`uRo&UUz|eD`rC@dlUX<$}|G&t&On zQ|0dz`*JCC2x)%Iu{zgpWzpP{ZKQZqy^#H}r8=J(L(?l|&Oh2Ua!)aBqM&J!CVgEn zePl#KOaQn2Sle$+k=j*H@vQ{wi8u(U65myFtmv-FT63Gr7*1xF8Te_5tUHn{ZYCml@m? zIXH-0JFYwh)Wi?4uzx?2s}~y!4tu>zFcHBVKwSQtx!X@kL3@U4%Bc9{zxf?J9Q*(7 zc5PT(^Yimn^Gux^lax6i@tmvam^t8`aVbsTC-zc-!-KJR85-xF9$e9yA9}2s`enK$ zPj9*<&NS??SHB=04Chemn-v#wUN_n=GznIW78QgF3rb#!zw*Iu>OtPL?aD7(rTi1l zR8srhmF(%<#}P2~KcDU?9Fi}O4PiCQl5)50S`tdic5rf_qHYsSo6k4J+Vg+{VnUu{S2d%ISIZm1HeWhY0KI&&7~K z^`FY3YvmCT!DC(4&QK$zL2;{A$~{o)8FL^N6P@+QM_YdU4Y)Et4L02J<}>t8n?m;p zu8gnFcQ@0$1;B`IvUqSAskpq?)(XZy*g`l{uYVri&1-sj%W(UFfTNn*#1}Q5@Z(sT zZ?`7>VXReHI%vFfrh+-^T!hUD0>=xRv0&aY>I0O@mi>u@a{o=t-d7bddp-mUuv;bc zT+9@~%%pku#g{c`_5=aw~GU{|PF3Bq+W7sJs*nFF`_WdF8>!Zwl&!45~4n&l{ z+<2()LZXwoo#*rm&Gyp$6cR_W<%m_G*`_m{7KHid7A^ah=){zcXr|YIN|~3=vy5C~ zW4zu2gG>*6Knmpqz6AhL|3rL#W>F;@Bv309I1IeMEKe#^%Khaf?F3C=#KxNRiK$lZ za&Aw#`&|ZMyH^%N|EQ^@<}-BsM9^J+_xq)cn@v`$)mJV1n-5Gpv{ckv!e|O8UgCYA zv61{Xr%JWnYkf?*xrLRe+1d09iEk&1H4oJ5J*M5&7IE9iN-{%t`O9m|&0&Pme3i)V z9t(@k9?H#$lL{#cnIeIl+O%W2T9;E)`PvgL_RfbD^ndZrlkrwc@$mVNHwI=wD;%fN z9IF%&?(^{tWuy!{OeLhpL4zE^hPREdVU^X;V+M>+1_s^m%Qlw zys*Kq{cWuAW=yJtV(i)d&A+*6RMEy2wsg*p2)^Yed-!mHvddetllSNx(Wc6(HH$cV zDKvbugiw(|bM_~E%q|gPlVG=Rm$PTZ9q(M35ZtmReqmvo;qKckv!nbB-_DHT(WQRV3&ZhMX=O;E?GEKtn!XW>LqM41BQX zbsVqa9T}qQ>p$(SJ5>}Iqat*+a5v^e1#|iAN$v}TLz1|jKiG6%d=CGxc{bU4smo%t z&a=E_gp0#pyW?Wg?2l*n3rqC6r*p z{>K6-rMHU)fVvZf5aCqdB6&^9u!YIRQEaX;UuZBdv2UI^!*}2VL&Zh7UZPZdJ}?+g zeoCLJduic*V&&W>hP_WW^X=A>*huwQ$BQF5j7FX~hPkfOoz_2U8n!t@vQ>C7hVJbp zPv-Q~DR0wdA|CV#UVLbAVDWUGu9`tjRaDJ|-VF~9Rb4p~)36w&qImX-atex={-^xM zo_0sdz0!#=3#Q@ieF}WIp8k8@{cp!r-j06=d0X9ho)pk*vRTvJvH>!lirnq{Hc)-} z@}=+9h^aSgxIHh~%PTa4Pu0$`w0|7SfBt{AG4b)Sii%GwRs=2eT6W^ZeZM~TG#g2N zUn$Ao@AkS|*8f!5`N1FT`RzWH{7JjBxg;chg=2ckMp)U68Th zATYaK+P^M-ZPe8rE52)P48D8I@cH}oN4(dc>EENjzo+lGtl)M2y#{xC10TQd{JqRp z(BDs5-dD<8x`QbeEM%ZjTSs*uP`$c3?XISTPBQe|;^PKJU`P9Z7*fJ3*=A z_vGKl%JrKy|8@$X;?(+XXJ|!NK+wLrmTyuMa zVqERxw-qL#>8r1t;wbO=n+j~i)OVXvP)aN$oyxR`V|MosQGx@f7_Eb)j6G3_Ryz`G8ZFW2U z({}eUReO8WY+3PmOSNZ9e}&(t3456nahlb;qoKX;Pn|GUlJk@vSPVr!MJiSDMN!`EU`aF>740o}u;R)pX6d9$ zEvvn?o&j}`P)u5aipjqGzGZ?68y<7@?EbrNM$P~D%d)^3>ogyUQdgOe&CFUg?a%jq zNi)g6X7TOb>dMyl-F%ph0o)>5t3kzC)$^~w!SuIU8$MiC{dnw8`J%eLb00rojymdB z`a0+Ow;jMnIq}YBXsiK7+D{>12buU{6&C)A7Y;a+t_yhT0kGekqM-sah6L9F-8zpX z??DwSC@Wl0a8?A)pOCDZ!{h@G?kK_S2T7n?!6S#bmEiEW!-B?iG8`&!0@yn^5#T|N zW0`num~prUpGzFvfVEYd6ZXw<_{|f@cpynel4CIfj@KB82;nyztHBKp%;!mue&C7? wL4k}33nySTlYn`^jtm>g(GTq0^&I%mtUiIc?{3-V90nlpboFyt=akR{0M$)#4FCWD literal 0 HcmV?d00001 From 3b2b573365edcc6b02c34edb6c4d1954a7028bd5 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Wed, 15 Jan 2025 22:46:23 -0500 Subject: [PATCH 02/27] Place venv in /root when using uv.lock as well (#3062) Signed-off-by: Thomas J. Fan --- flytekit/image_spec/default_builder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py index 96b9acbe9e..20e624d27f 100644 --- a/flytekit/image_spec/default_builder.py +++ b/flytekit/image_spec/default_builder.py @@ -23,15 +23,17 @@ UV_LOCK_INSTALL_TEMPLATE = Template( """\ +WORKDIR /root RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \ --mount=from=uv,source=/uv,target=/usr/bin/uv \ --mount=type=bind,target=uv.lock,src=uv.lock \ --mount=type=bind,target=pyproject.toml,src=pyproject.toml \ uv sync $PIP_INSTALL_ARGS +WORKDIR / # Update PATH and UV_PYTHON to point to the venv created by uv sync -ENV PATH="/.venv/bin:$$PATH" \ - UV_PYTHON=/.venv/bin/python +ENV PATH="/root/.venv/bin:$$PATH" \ + UV_PYTHON=/root/.venv/bin/python """ ) @@ -407,7 +409,7 @@ def _build_image(self, image_spec: ImageSpec, *, push: bool = True) -> str: if value is not None and name not in self._SUPPORTED_IMAGE_SPEC_PARAMETERS and not name.startswith("_") ] if unsupported_parameters: - msg = f"The following parameters are unsupported and ignored: " f"{unsupported_parameters}" + msg = f"The following parameters are unsupported and ignored: {unsupported_parameters}" warnings.warn(msg, UserWarning, stacklevel=2) with tempfile.TemporaryDirectory() as tmp_dir: From 4b50681badb3e3ce2fd641f775b74ac9ff7014f1 Mon Sep 17 00:00:00 2001 From: Rafael Raposo <100569684+RRap0so@users.noreply.github.com> Date: Thu, 16 Jan 2025 08:11:17 +0100 Subject: [PATCH 03/27] Make FlyteUserRuntimeException to return error_code in Container Error (#3059) * Make FlyteUserRuntimeException to return error_code in the ContainerError Signed-off-by: Rafael Ribeiro Raposo --- flytekit/bin/entrypoint.py | 2 +- flytekit/exceptions/user.py | 4 +++ .../unit/bin/test_python_entrypoint.py | 33 +++++++++++++++++++ tests/flytekit/unit/exceptions/test_user.py | 10 ++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 49103319d0..cc0f7bafbe 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -266,7 +266,7 @@ def _dispatch_execute( exc_str = get_traceback_str(e) output_file_dict[error_file_name] = _error_models.ErrorDocument( _error_models.ContainerError( - code="USER", + code=e.error_code, message=exc_str, kind=kind, origin=_execution_models.ExecutionError.ErrorKind.USER, diff --git a/flytekit/exceptions/user.py b/flytekit/exceptions/user.py index 3413d172ff..af4dbf63c6 100644 --- a/flytekit/exceptions/user.py +++ b/flytekit/exceptions/user.py @@ -28,6 +28,10 @@ def __init__(self, exc_value: Exception): def value(self): return self._exc_value + @property + def error_code(self): + return self._ERROR_CODE + class FlyteTypeException(FlyteUserException, TypeError): _ERROR_CODE = "USER:TypeError" diff --git a/tests/flytekit/unit/bin/test_python_entrypoint.py b/tests/flytekit/unit/bin/test_python_entrypoint.py index 323770bed7..8a1709d668 100644 --- a/tests/flytekit/unit/bin/test_python_entrypoint.py +++ b/tests/flytekit/unit/bin/test_python_entrypoint.py @@ -169,6 +169,7 @@ def verify_output(*args, **kwargs): assert error_filename_base.startswith("error-") uuid.UUID(hex=error_filename_base[6:], version=4) assert error_filename_ext == ".pb" + assert container_error.code == "USER:RuntimeError" mock_write_to_file.side_effect = verify_output _dispatch_execute(ctx, lambda: python_task, "inputs path", "outputs prefix") @@ -991,3 +992,35 @@ def t1(a: typing.List[int]) -> typing.List[typing.List[str]]: assert lit.literals["o0"].HasField("offloaded_metadata") == False else: assert False, f"Unexpected file {ff}" + + +@mock.patch("flytekit.core.utils.load_proto_from_file") +@mock.patch("flytekit.core.data_persistence.FileAccessProvider.get_data") +@mock.patch("flytekit.core.data_persistence.FileAccessProvider.put_data") +@mock.patch("flytekit.core.utils.write_proto_to_file") +def test_dispatch_execute_custom_error_code_with_flyte_user_runtime_exception(mock_write_to_file, mock_upload_dir, mock_get_data, mock_load_proto): + class CustomException(FlyteUserRuntimeException): + _ERROR_CODE = "CUSTOM_ERROR_CODE" + + mock_get_data.return_value = True + mock_upload_dir.return_value = True + + ctx = context_manager.FlyteContext.current_context() + with context_manager.FlyteContextManager.with_context( + ctx.with_execution_state( + ctx.execution_state.with_params(mode=context_manager.ExecutionState.Mode.TASK_EXECUTION) + ) + ) as ctx: + python_task = mock.MagicMock() + python_task.dispatch_execute.side_effect = CustomException("custom error") + + empty_literal_map = _literal_models.LiteralMap({}).to_flyte_idl() + mock_load_proto.return_value = empty_literal_map + + def verify_output(*args, **kwargs): + assert isinstance(args[0], ErrorDocument) + assert args[0].error.code == "CUSTOM_ERROR_CODE" + + mock_write_to_file.side_effect = verify_output + _dispatch_execute(ctx, lambda: python_task, "inputs path", "outputs prefix") + assert mock_write_to_file.call_count == 1 diff --git a/tests/flytekit/unit/exceptions/test_user.py b/tests/flytekit/unit/exceptions/test_user.py index fedacbebc6..15ae795250 100644 --- a/tests/flytekit/unit/exceptions/test_user.py +++ b/tests/flytekit/unit/exceptions/test_user.py @@ -11,6 +11,16 @@ def test_flyte_user_exception(): assert type(e).error_code == "USER:Unknown" assert isinstance(e, base.FlyteException) +def test_flyte_user_runtime_exception(): + try: + base_exn = Exception("everywhere is bad") + raise user.FlyteUserRuntimeException("bad") from base_exn + except Exception as e: + assert str(e) == "USER:RuntimeError: error=bad, cause=everywhere is bad" + assert isinstance(type(e), base._FlyteCodedExceptionMetaclass) + assert type(e).error_code == "USER:RuntimeError" + assert isinstance(e, base.FlyteException) + assert isinstance(e, user.FlyteUserRuntimeException) def test_flyte_type_exception(): try: From c3cfa8915e2237ac71575d1bccb4cff4fbb1d81c Mon Sep 17 00:00:00 2001 From: "Han-Ru Chen (Future-Outlier)" Date: Thu, 16 Jan 2025 15:43:18 +0800 Subject: [PATCH 04/27] Improve Type Engine Error Msg (#3063) * update Signed-off-by: Future-Outlier * ficx Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier * test Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier * udpate Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier * updatre Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier --- flytekit/core/type_engine.py | 11 ++++++++--- tests/flytekit/unit/core/test_type_hints.py | 9 ++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 29092035b7..2b4619540d 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -1518,9 +1518,14 @@ async def _literal_map_to_kwargs( TypeEngine.async_to_python_value(ctx, lm.literals[k], python_interface_inputs[k]) ) await asyncio.gather(*kwargs.values()) - except TypeTransformerFailedError as exc: - exc.args = (f"Error converting input '{k}' at position {i}:\n {exc.args[0]}",) - raise + except Exception as e: + raise TypeTransformerFailedError( + f"Error converting input '{k}' at position {i}:\n" + f"Literal value: {lm.literals[k]}\n" + f"Literal type: {literal_types}\n" + f"Expected Python type: {python_interface_inputs[k]}\n" + f"Exception: {e}" + ) kwargs = {k: v.result() for k, v in kwargs.items() if v is not None} return kwargs diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 1074423baf..ec977020b0 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -1896,12 +1896,11 @@ def wf2(a: typing.Union[int, str]) -> typing.Union[int, str]: with pytest.raises( TypeError, - match=re.escape( - f"Error encountered while converting inputs of '{exec_prefix}tests.flytekit.unit.core.test_type_hints.t2':\n" - r" Cannot convert from Flyte Serialized object (Literal):" - ), + match=( + rf"Error encountered while converting inputs of '{exec_prefix}tests\.flytekit\.unit\.core\.test_type_hints\.t2':\n\s+Error converting input 'a' at position 0:" + ), ): - assert wf2(a="2") == "2" + wf2(a="2") # Removed assert as it was not necessary for the exception to be raised def test_optional_type(): From 0bb8bad5954b1c5edd22da4362c605b6b3f84d3b Mon Sep 17 00:00:00 2001 From: Vincent Chen <62143443+mao3267@users.noreply.github.com> Date: Fri, 17 Jan 2025 22:39:22 +0800 Subject: [PATCH 05/27] [Flytekit] Separate remote signal functions (#2933) * feat: separate remote signal functions Signed-off-by: mao3267 * refactor: make lint Signed-off-by: mao3267 * test: add integration test for separated signal functions Signed-off-by: mao3267 * fix: register workflow to admin Signed-off-by: mao3267 * fix: integration test and approve function Signed-off-by: mao3267 * fix: remove approve node output Signed-off-by: mao3267 * fix: replace single sleep command to retry statement Signed-off-by: mao3267 * fix: update comments Signed-off-by: mao3267 * fix: simplify duplicate retry operations Signed-off-by: mao3267 --------- Signed-off-by: mao3267 --- flytekit/remote/remote.py | 87 +++++++++++++++++++ .../integration/remote/test_remote.py | 33 +++++++ .../remote/workflows/basic/signal_test.py | 17 ++++ 3 files changed, 137 insertions(+) create mode 100644 tests/flytekit/integration/remote/workflows/basic/signal_test.py diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 09c44f949b..6bff52ade2 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -632,6 +632,93 @@ def list_signals( s = resp.signals return s + def approve(self, signal_id: str, execution_name: str, project: str = None, domain: str = None): + """ + :param signal_id: The name of the signal, this is the key used in the approve() or wait_for_input() call. + :param execution_name: The name of the execution. This is the tail-end of the URL when looking + at the workflow execution. + :param project: The execution project, will default to the Remote's default project. + :param domain: The execution domain, will default to the Remote's default domain. + """ + + wf_exec_id = WorkflowExecutionIdentifier( + project=project or self.default_project, domain=domain or self.default_domain, name=execution_name + ) + + lt = TypeEngine.to_literal_type(bool) + true_literal = TypeEngine.to_literal(self.context, True, bool, lt) + + req = SignalSetRequest( + id=SignalIdentifier(signal_id, wf_exec_id).to_flyte_idl(), value=true_literal.to_flyte_idl() + ) + + # Response is empty currently, nothing to give back to the user. + self.client.set_signal(req) + + def reject(self, signal_id: str, execution_name: str, project: str = None, domain: str = None): + """ + :param signal_id: The name of the signal, this is the key used in the approve() or wait_for_input() call. + :param execution_name: The name of the execution. This is the tail-end of the URL when looking + at the workflow execution. + :param project: The execution project, will default to the Remote's default project. + :param domain: The execution domain, will default to the Remote's default domain. + """ + + wf_exec_id = WorkflowExecutionIdentifier( + project=project or self.default_project, domain=domain or self.default_domain, name=execution_name + ) + + lt = TypeEngine.to_literal_type(bool) + false_literal = TypeEngine.to_literal(self.context, False, bool, lt) + + req = SignalSetRequest( + id=SignalIdentifier(signal_id, wf_exec_id).to_flyte_idl(), value=false_literal.to_flyte_idl() + ) + + # Response is empty currently, nothing to give back to the user. + self.client.set_signal(req) + + def set_input( + self, + signal_id: str, + execution_name: str, + value: typing.Union[literal_models.Literal, typing.Any], + project=None, + domain=None, + python_type=None, + literal_type=None, + ): + """ + :param signal_id: The name of the signal, this is the key used in the approve() or wait_for_input() call. + :param execution_name: The name of the execution. This is the tail-end of the URL when looking + at the workflow execution. + :param value: This is either a Literal or a Python value which FlyteRemote will invoke the TypeEngine to + convert into a Literal. This argument is only value for wait_for_input type signals. + :param project: The execution project, will default to the Remote's default project. + :param domain: The execution domain, will default to the Remote's default domain. + :param python_type: Provide a python type to help with conversion if the value you provided is not a Literal. + :param literal_type: Provide a Flyte literal type to help with conversion if the value you provided + is not a Literal + """ + + wf_exec_id = WorkflowExecutionIdentifier( + project=project or self.default_project, domain=domain or self.default_domain, name=execution_name + ) + if isinstance(value, Literal): + logger.debug(f"Using provided {value} as existing Literal value") + lit = value + else: + lt = literal_type or ( + TypeEngine.to_literal_type(python_type) if python_type else TypeEngine.to_literal_type(type(value)) + ) + lit = TypeEngine.to_literal(self.context, value, python_type or type(value), lt) + logger.debug(f"Converted {value} to literal {lit} using literal type {lt}") + + req = SignalSetRequest(id=SignalIdentifier(signal_id, wf_exec_id).to_flyte_idl(), value=lit.to_flyte_idl()) + + # Response is empty currently, nothing to give back to the user. + self.client.set_signal(req) + def set_signal( self, signal_id: str, diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index a559ffe09f..82c18b3c50 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -866,3 +866,36 @@ def test_attr_access_sd(): url = urlparse(remote_file_path) bucket, key = url.netloc, url.path.lstrip("/") file_transfer.delete_file(bucket=bucket, key=key) + +def test_signal_approve_reject(register): + from flytekit.models.types import LiteralType, SimpleType + from time import sleep + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + conditional_wf = remote.fetch_workflow(name="basic.signal_test.signal_test_wf", version=VERSION) + + execution = remote.execute(conditional_wf, inputs={"data": [1.0, 2.0, 3.0, 4.0, 5.0]}) + + def retry_operation(operation): + max_retries = 10 + for _ in range(max_retries): + try: + operation() + break + except Exception: + sleep(1) + + retry_operation(lambda: remote.set_input("title-input", execution.id.name, value="my report", project=PROJECT, domain=DOMAIN, python_type=str, literal_type=LiteralType(simple=SimpleType.STRING))) + retry_operation(lambda: remote.approve("review-passes", execution.id.name, project=PROJECT, domain=DOMAIN)) + + remote.wait(execution=execution, timeout=datetime.timedelta(minutes=5)) + assert execution.outputs["o0"] == {"title": "my report", "data": [1.0, 2.0, 3.0, 4.0, 5.0]} + + with pytest.raises(FlyteAssertion, match="Outputs could not be found because the execution ended in failure"): + execution = remote.execute(conditional_wf, inputs={"data": [1.0, 2.0, 3.0, 4.0, 5.0]}) + + retry_operation(lambda: remote.set_input("title-input", execution.id.name, value="my report", project=PROJECT, domain=DOMAIN, python_type=str, literal_type=LiteralType(simple=SimpleType.STRING))) + retry_operation(lambda: remote.reject("review-passes", execution.id.name, project=PROJECT, domain=DOMAIN)) + + remote.wait(execution=execution, timeout=datetime.timedelta(minutes=5)) + assert execution.outputs["o0"] == {"title": "my report", "data": [1.0, 2.0, 3.0, 4.0, 5.0]} diff --git a/tests/flytekit/integration/remote/workflows/basic/signal_test.py b/tests/flytekit/integration/remote/workflows/basic/signal_test.py new file mode 100644 index 0000000000..c2771ffdfd --- /dev/null +++ b/tests/flytekit/integration/remote/workflows/basic/signal_test.py @@ -0,0 +1,17 @@ +from datetime import timedelta +from flytekit import task, workflow, wait_for_input, approve, conditional +import typing + +@task +def reporting_wf(title_input: str, data: typing.List[float]) -> dict: + return {"title": title_input, "data": data} + +@workflow +def signal_test_wf(data: typing.List[float]) -> dict: + title_input = wait_for_input(name="title-input", timeout=timedelta(hours=1), expected_type=str) + + # Define a "review-passes" approve node so that a human can review + # the title before finalizing it. + approve(upstream_item=title_input, name="review-passes", timeout=timedelta(hours=1)) + + return reporting_wf(title_input, data) From 30088c2c970aff0b76c0bdd4db72b3c3e1d6fd18 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 17 Jan 2025 14:21:57 -0500 Subject: [PATCH 06/27] Only copy over cat-certificates.crt if it does not exist in base image (#3067) * Do not copy over ca-certifcates.crt if the base image has one Signed-off-by: Thomas J. Fan * Only copy over cat-certificates.crt if it does not exist in base image Signed-off-by: Thomas J. Fan --------- Signed-off-by: Thomas J. Fan --- flytekit/image_spec/default_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py index 20e624d27f..29583a4386 100644 --- a/flytekit/image_spec/default_builder.py +++ b/flytekit/image_spec/default_builder.py @@ -87,7 +87,9 @@ USER root $APT_INSTALL_COMMAND -COPY --from=micromamba /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +RUN --mount=from=micromamba,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/ca-certificates.crt \ + [ -f /etc/ssl/certs/ca-certificates.crt ] || \ + mkdir -p /etc/ssl/certs/ && cp /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt RUN id -u flytekit || useradd --create-home --shell /bin/bash flytekit RUN chown -R flytekit /root && chown -R flytekit /home From a46593242295e6aa7a60e07515eba651f848dc3f Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 17 Jan 2025 16:38:56 -0800 Subject: [PATCH 07/27] Update eager task launching & monitoring (#3042) Signed-off-by: Yee Hing Tong --- flytekit/core/context_manager.py | 6 +- flytekit/core/promise.py | 5 +- flytekit/core/worker_queue.py | 410 ++++++++++-------- tests/flytekit/unit/core/test_worker_queue.py | 230 ++++++++-- 4 files changed, 430 insertions(+), 221 deletions(-) diff --git a/flytekit/core/context_manager.py b/flytekit/core/context_manager.py index 29d86aa8fc..ab19939522 100644 --- a/flytekit/core/context_manager.py +++ b/flytekit/core/context_manager.py @@ -18,6 +18,7 @@ import pathlib import signal import tempfile +import threading import traceback import typing from contextlib import contextmanager @@ -994,7 +995,10 @@ def main_signal_handler(signum: int, frame: FrameType): handler(signum, frame) exit(1) - signal.signal(signal.SIGINT, main_signal_handler) + # This initialize function is also called by other threads (since the context manager lives in a ContextVar) + # so we should not run this if we're not the main thread. + if threading.current_thread().name == threading.main_thread().name: + signal.signal(signal.SIGINT, main_signal_handler) # Note we use the SdkWorkflowExecution object purely for formatting into the ex:project:domain:name format users # are already acquainted with diff --git a/flytekit/core/promise.py b/flytekit/core/promise.py index 3db02175b5..d64f2461e5 100644 --- a/flytekit/core/promise.py +++ b/flytekit/core/promise.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import collections import datetime import typing @@ -1436,9 +1435,7 @@ async def async_flyte_entity_call_handler( # for both nested eager, async, and sync tasks, submit to the informer. if not ctx.worker_queue: raise AssertionError("Worker queue missing, must be set when trying to execute tasks in an eager workflow") - loop = asyncio.get_running_loop() - fut = ctx.worker_queue.add(loop, entity, input_kwargs=kwargs) - result = await fut + result = await ctx.worker_queue.add(entity, input_kwargs=kwargs) return result # eager local execution, and all other call patterns are handled by the sync version diff --git a/flytekit/core/worker_queue.py b/flytekit/core/worker_queue.py index df35e473ab..6b95a748b4 100644 --- a/flytekit/core/worker_queue.py +++ b/flytekit/core/worker_queue.py @@ -5,9 +5,11 @@ import hashlib import re import threading +import time import typing -from concurrent.futures import Future +import uuid from dataclasses import dataclass +from enum import Enum from flytekit.configuration import ImageConfig, SerializationSettings from flytekit.core.base_task import PythonTask @@ -26,6 +28,7 @@ from flytekit.remote.remote_callable import RemoteEntity RunnableEntity = typing.Union[WorkflowBase, LaunchPlan, PythonTask, ReferenceEntity, RemoteEntity] + from flytekit.core.interface import Interface from flytekit.remote import FlyteRemote, FlyteWorkflowExecution @@ -66,116 +69,78 @@ """ +class ItemStatus(Enum): + PENDING = "Pending" + RUNNING = "Running" + SUCCESS = "Success" + FAILED = "Failed" + + @dataclass +class Update: + # The item to update + work_item: WorkItem + idx: int + + # fields in the item to update + status: typing.Optional[ItemStatus] = None + wf_exec: typing.Optional[FlyteWorkflowExecution] = None + error: typing.Optional[BaseException] = None + + +@dataclass(unsafe_hash=True) class WorkItem: + """ + This is a class to keep track of what the user requested. Since it captures the arguments that the user wants + to run the entity with, an arbitrary map, can't make this frozen. + """ + entity: RunnableEntity input_kwargs: dict[str, typing.Any] - fut: asyncio.Future result: typing.Any = None error: typing.Optional[BaseException] = None - + status: ItemStatus = ItemStatus.PENDING wf_exec: typing.Optional[FlyteWorkflowExecution] = None + python_interface: typing.Optional[Interface] = None + # This id is the key between this object and the Update object. The reason it's useful is because this + # dataclass is not hashable. + uuid: typing.Optional[uuid.UUID] = None - def set_result(self, result: typing.Any): - assert self.wf_exec is not None - developer_logger.debug(f"Setting result for {self.wf_exec.id.name} on thread {threading.current_thread().name}") - self.result = result - # need to convert from literals resolver to literals and then to python native. - self.fut._loop.call_soon_threadsafe(self.fut.set_result, result) - - def set_error(self, e: BaseException): - developer_logger.debug( - f"Setting error for {self.wf_exec.id.name if self.wf_exec else 'unstarted execution'}" - f" on thread {threading.current_thread().name} to {e}" - ) - self.error = e - self.fut._loop.call_soon_threadsafe(self.fut.set_exception, e) - - def set_exec(self, wf_exec: FlyteWorkflowExecution): - self.wf_exec = wf_exec + def __post_init__(self): + self.python_interface = self._get_python_interface() + self.uuid = uuid.uuid4() @property - def ready(self) -> bool: - return self.wf_exec is not None and (self.result is not None or self.error is not None) - - -class Informer(typing.Protocol): - def watch(self, work: WorkItem): ... - - -class PollingInformer: - def __init__(self, remote: FlyteRemote, loop: asyncio.AbstractEventLoop): - self.remote: FlyteRemote = remote - self.__loop = loop - - async def watch_one(self, work: WorkItem): - assert work.wf_exec is not None - logger.debug(f"Starting watching execution {work.wf_exec.id} on {threading.current_thread().name}") - while True: - # not really async but pretend it is for now, change to to_thread in the future. - developer_logger.debug(f"Looping on {work.wf_exec.id.name}") - self.remote.sync_execution(work.wf_exec) - if work.wf_exec.is_done: - developer_logger.debug(f"Execution {work.wf_exec.id.name} is done.") - break - await asyncio.sleep(2) - - # set results - # but first need to convert from literals resolver to literals and then to python native. - if work.wf_exec.closure.phase == WorkflowExecutionPhase.SUCCEEDED: - from flytekit.core.interface import Interface - from flytekit.core.type_engine import TypeEngine - - if not work.entity.python_interface: - for k, _ in work.entity.interface.outputs.items(): - if not re.match(standard_output_format, k): - raise AssertionError( - f"Entity without python interface found, and output name {k} does not match standard format o[0-9]+" - ) - - num_outputs = len(work.entity.interface.outputs) - python_outputs_interface: typing.Dict[str, typing.Type] = {} - # Iterate in order so that we add to the interface in the correct order - for i in range(num_outputs): - key = f"o{i}" - if key not in work.entity.interface.outputs: - raise AssertionError( - f"Output name {key} not found in outputs {[k for k in work.entity.interface.outputs.keys()]}" - ) - var_type = work.entity.interface.outputs[key].type - python_outputs_interface[key] = TypeEngine.guess_python_type(var_type) - py_iface = Interface(inputs=typing.cast(dict[str, typing.Type], {}), outputs=python_outputs_interface) - else: - py_iface = work.entity.python_interface - - results = work.wf_exec.outputs.as_python_native(py_iface) + def is_in_terminal_state(self) -> bool: + return self.status == ItemStatus.SUCCESS or self.status == ItemStatus.FAILED - work.set_result(results) - elif work.wf_exec.closure.phase == WorkflowExecutionPhase.FAILED: - from flytekit.exceptions.eager import EagerException + def _get_python_interface(self) -> Interface: + from flytekit.core.interface import Interface + from flytekit.core.type_engine import TypeEngine - exc = EagerException(f"Error executing {work.entity.name} with error: {work.wf_exec.closure.error}") - work.set_error(exc) + if self.entity.python_interface: + return self.entity.python_interface - def watch(self, work: WorkItem): - coro = self.watch_one(work) - # both run_coroutine_threadsafe and self.__loop.create_task seem to work, but the first makes - # more sense in case *this* thread is ever different than the thread that self.__loop is running on. - f = asyncio.run_coroutine_threadsafe(coro, self.__loop) - - developer_logger.debug(f"Started watch with future {f}") + for k, _ in self.entity.interface.outputs.items(): + if not re.match(standard_output_format, k): + raise AssertionError( + f"Entity without python interface found, and output name {k} does not match standard format o[0-9]+" + ) - def cb(fut: Future): - """ - This cb takes care of any exceptions that might be thrown in the watch_one coroutine - Note: This is a concurrent Future not an asyncio Future - """ - e = fut.exception() - if e: - logger.error(f"Error in watch for {work.entity.name} with {work.input_kwargs}: {e}") - work.set_error(e) + num_outputs = len(self.entity.interface.outputs) + python_outputs_interface: typing.Dict[str, typing.Type] = {} + # Iterate in order so that we add to the interface in the correct order + for i in range(num_outputs): + key = f"o{i}" + if key not in self.entity.interface.outputs: + raise AssertionError( + f"Output name {key} not found in outputs {[k for k in self.entity.interface.outputs.keys()]}" + ) + var_type = self.entity.interface.outputs[key].type + python_outputs_interface[key] = TypeEngine.guess_python_type(var_type) + py_iface = Interface(inputs=typing.cast(dict[str, typing.Type], {}), outputs=python_outputs_interface) - f.add_done_callback(cb) + return py_iface # A flag to ensure the handler runs only once @@ -183,29 +148,43 @@ def cb(fut: Future): class Controller: + """ + This controller object is responsible for kicking off and monitoring executions against a Flyte Admin endpoint + using a FlyteRemote object. It is used only for running eager tasks. It exposes one async method, `add`, which + should be called by the eager task to run a sub-flyte-entity (task, workflow, or a nested eager task). + + The controller maintains a dictionary of entries, where each entry is a list of WorkItems. They are maintained + in a list because the number of times and order that each task (or subwf, lp) is called affects the execution name + which is consistently hashed. + + After calling `add`, a background thread is started to reconcile the state of this dictionary of WorkItem entries. + Executions that should be kicked off will be kicked off, and ones that are running will be checked. This runs + in a loop similar to a controller loop in a k8s operator. + """ + def __init__(self, remote: FlyteRemote, ss: SerializationSettings, tag: str, root_tag: str, exec_prefix: str): logger.debug( f"Creating Controller for eager execution with {remote.config.platform.endpoint}," f" {tag=}, {root_tag=}, {exec_prefix=} and ss: {ss}" ) - # Set up things for this controller to operate - from flytekit.utils.asyn import _selector_policy - - with _selector_policy(): - self.__loop = asyncio.new_event_loop() - self.__loop.set_exception_handler(self.exc_handler) - self.__runner_thread: threading.Thread = threading.Thread( - target=self._execute, daemon=True, name="controller-loop-runner" - ) - self.__runner_thread.start() - atexit.register(self._close) - # Things for actually kicking off and monitoring self.entries: typing.Dict[str, typing.List[WorkItem]] = {} - self.informer = PollingInformer(remote=remote, loop=self.__loop) self.remote = remote self.ss = ss self.exec_prefix = exec_prefix + self.entries_lock = threading.Lock() + from flytekit.core.context_manager import FlyteContextManager + + # Import this to ensure context is loaded... python is reloading this module because its in a different thread + FlyteContextManager.current_context() + + self.stopping_condition = threading.Event() + # Things for actually kicking off and monitoring + self.__runner_thread: threading.Thread = threading.Thread( + target=self._execute, daemon=True, name="controller-thread" + ) + self.__runner_thread.start() + atexit.register(self._close, stopping_condition=self.stopping_condition, runner=self.__runner_thread) # Executions should be tracked in the following way: # a) you should be able to list by label, all executions generated by the current eager task, @@ -226,20 +205,111 @@ def __init__(self, remote: FlyteRemote, ss: SerializationSettings, tag: str, roo self.tag = tag self.root_tag = root_tag - def _close(self) -> None: - if self.__loop: - self.__loop.stop() - @staticmethod - def exc_handler(loop, context): - logger.error(f"Caught exception in loop {loop} with context {context}") + def _close(stopping_condition: threading.Event, runner: threading.Thread) -> None: + stopping_condition.set() + logger.debug("Set background thread to stop, awaiting...") + runner.join() + + def reconcile_one(self, update: Update): + """ + This is responsible for processing one work item. Will launch, update, set error on the update object + Any errors are captured in the update object. + """ + try: + item = update.work_item + if item.wf_exec is None: + logger.warning(f"reconcile should launch for {id(item)} entity name: {item.entity.name}") + wf_exec = self.launch_execution(update.work_item, update.idx) + update.wf_exec = wf_exec + update.status = ItemStatus.RUNNING + else: + if not item.wf_exec.is_done: + update.status = ItemStatus.RUNNING + # Technically a mutating operation, but let's pretend it's not + update.wf_exec = self.remote.sync_execution(item.wf_exec) + if update.wf_exec.closure.phase == WorkflowExecutionPhase.SUCCEEDED: + update.status = ItemStatus.SUCCESS + elif update.wf_exec.closure.phase == WorkflowExecutionPhase.FAILED: + update.status = ItemStatus.FAILED + else: + developer_logger.debug(f"Execution {item.wf_exec.id.name} is done, item is {item.status}") + + except Exception as e: + logger.error( + f"Error launching execution for {update.work_item.entity.name} with {update.work_item.input_kwargs}: {e}" + ) + update.error = e + update.status = ItemStatus.FAILED + + def _get_update_items(self) -> typing.Dict[uuid.UUID, Update]: + with self.entries_lock: + update_items: typing.Dict[uuid.UUID, Update] = {} + for entity_name, items in self.entries.items(): + for idx, item in enumerate(items): + # Only process items that need it + if item.status == ItemStatus.SUCCESS or item.status == ItemStatus.FAILED: + continue + update = Update(work_item=item, idx=idx) + update_items[typing.cast(uuid.UUID, item.uuid)] = update + return update_items + + def _apply_updates(self, update_items: typing.Dict[uuid.UUID, Update]) -> None: + with self.entries_lock: + for entity_name, items in self.entries.items(): + for item in items: + if item.uuid in update_items: + update = update_items[typing.cast(uuid.UUID, item.uuid)] + item.wf_exec = update.wf_exec + assert update.status is not None + item.status = update.status + if update.status == ItemStatus.SUCCESS: + assert update.wf_exec is not None + item.result = update.wf_exec.outputs.as_python_native(item.python_interface) + elif update.status == ItemStatus.FAILED: + # If update object already has an error, then use that, otherwise look for one in the + # execution closure. + if update.error: + item.error = update.error + else: + from flytekit.exceptions.eager import EagerException + + assert update.wf_exec is not None + + exc = EagerException( + f"Error executing {update.work_item.entity.name} with error:" + f" {update.wf_exec.closure.error}" + ) + item.error = exc + + # otherwise it's still pending or running + + def _poll(self) -> None: + # This needs to be a while loop that runs forever, + while True: + if self.stopping_condition.is_set(): + developer_logger.debug("Controller thread stopping detected, quitting poll loop") + break + # Gather all items that need processing + update_items = self._get_update_items() + + # Actually call for updates outside of the lock. + # Currently this happens one at a time, but only because the API only supports one at a time. + for up in update_items.values(): + self.reconcile_one(up) + + # Take the lock again and apply all the updates + self._apply_updates(update_items) + + # This is a blocking call so we don't hit the API too much. + time.sleep(2) def _execute(self) -> None: - loop = self.__loop try: - loop.run_forever() - finally: - logger.error("Controller event loop stopped.") + self._poll() + except Exception as e: + logger.error(f"Error in eager execution processor: {e}") + exit(1) def get_labels(self) -> Labels: """ @@ -272,66 +342,62 @@ def get_execution_name(self, entity: RunnableEntity, idx: int, input_kwargs: dic exec_name = _dnsify(exec_name) return exec_name - def launch_and_start_watch(self, wi: WorkItem, idx: int): - """This function launches executions. This is called via the loop, so it needs exception handling""" - try: - if wi.result is None and wi.error is None: - l = self.get_labels() - e = self.get_env() - options = Options(labels=l) - exec_name = self.get_execution_name(wi.entity, idx, wi.input_kwargs) - logger.info(f"Generated execution name {exec_name} for {idx}th call of {wi.entity.name}") - from flytekit.remote.remote_callable import RemoteEntity - - if isinstance(wi.entity, RemoteEntity): - version = wi.entity.id.version - else: - version = self.ss.version - - # todo: if the execution already exists, remote.execute will return that execution. in the future - # we can add input checking to make sure the inputs are indeed a match. - wf_exec = self.remote.execute( - entity=wi.entity, - execution_name=exec_name, - inputs=wi.input_kwargs, - version=version, - image_config=self.ss.image_config, - options=options, - envs=e, - ) - logger.info(f"Successfully started execution {wf_exec.id.name}") - wi.set_exec(wf_exec) - - # if successful then start watch on the execution - self.informer.watch(wi) + def launch_execution(self, wi: WorkItem, idx: int) -> FlyteWorkflowExecution: + """This function launches executions.""" + logger.warning(f"Launching execution for {wi.entity.name} {idx=} with {wi.input_kwargs}") + if wi.result is None and wi.error is None: + l = self.get_labels() + e = self.get_env() + options = Options(labels=l) + exec_name = self.get_execution_name(wi.entity, idx, wi.input_kwargs) + logger.info(f"Generated execution name {exec_name} for {idx}th call of {wi.entity.name}") + from flytekit.remote.remote_callable import RemoteEntity + + if isinstance(wi.entity, RemoteEntity): + version = wi.entity.id.version else: - raise AssertionError( - "This launch function should not be invoked for work items already" " with result or error" - ) - except Exception as e: - # all exceptions get registered onto the future. - logger.error(f"Error launching execution for {wi.entity.name} with {wi.input_kwargs}") - wi.set_error(e) + assert self.ss.version + version = self.ss.version + + # todo: if the execution already exists, remote.execute will return that execution. in the future + # we can add input checking to make sure the inputs are indeed a match. + wf_exec = self.remote.execute( + entity=wi.entity, + execution_name=exec_name, + inputs=wi.input_kwargs, + version=version, + image_config=self.ss.image_config, + options=options, + envs=e, + ) + return wf_exec + else: + raise AssertionError( + "This launch function should not be invoked for work items already" " with result or error" + ) - def add( - self, task_loop: asyncio.AbstractEventLoop, entity: RunnableEntity, input_kwargs: dict[str, typing.Any] - ) -> asyncio.Future: + async def add(self, entity: RunnableEntity, input_kwargs: dict[str, typing.Any]) -> typing.Any: """ Add an entity along with the requested inputs to be submitted to Admin for running and return a future """ # need to also check to see if the entity has already been registered, and if not, register it. - fut = task_loop.create_future() - i = WorkItem(entity=entity, input_kwargs=input_kwargs, fut=fut) + i = WorkItem(entity=entity, input_kwargs=input_kwargs) - # For purposes of awaiting an execution, we don't need to keep track of anything, but doing so for Deck - if entity.name not in self.entries: - self.entries[entity.name] = [] - self.entries[entity.name].append(i) - idx = len(self.entries[entity.name]) - 1 + with self.entries_lock: + if entity.name not in self.entries: + self.entries[entity.name] = [] + self.entries[entity.name].append(i) - # trigger a run of the launching function. - self.__loop.call_soon_threadsafe(self.launch_and_start_watch, i, idx) - return fut + # wait for it to finish one way or another + while True: + developer_logger.debug(f"Watching id {id(i)}") + if i.status == ItemStatus.SUCCESS: + return i.result + elif i.status == ItemStatus.FAILED: + assert i.error is not None + raise i.error + else: + await asyncio.sleep(2) # Small delay to avoid busy-waiting def render_html(self) -> str: """Render the callstack as a deck presentation to be shown after eager workflow execution.""" @@ -355,7 +421,7 @@ def _entity_type(entity) -> str: for entity_name, items_list in self.entries.items(): for item in items_list: - if not item.ready: + if not item.is_in_terminal_state: logger.warning( f"Item for {item.entity.name} with inputs {item.input_kwargs}" f" isn't ready, skipping for deck rendering..." diff --git a/tests/flytekit/unit/core/test_worker_queue.py b/tests/flytekit/unit/core/test_worker_queue.py index 1c1cfb1446..0a934fc20a 100644 --- a/tests/flytekit/unit/core/test_worker_queue.py +++ b/tests/flytekit/unit/core/test_worker_queue.py @@ -1,16 +1,30 @@ import mock import pytest -import asyncio - +import datetime from flytekit.core.task import task from flytekit.remote.remote import FlyteRemote -from flytekit.core.worker_queue import Controller, WorkItem +from flytekit.core.worker_queue import Controller, WorkItem, ItemStatus, Update from flytekit.configuration import ImageConfig, LocalConfig, SerializationSettings from flytekit.utils.asyn import loop_manager +from flytekit.models.execution import ExecutionSpec, ExecutionClosure, ExecutionMetadata, NotificationList, Execution, AbortMetadata +from flytekit.models.core import identifier +from flytekit.models import common as common_models +from flytekit.models.core import execution +from flytekit.exceptions.eager import EagerException + + +def _mock_reconcile(update: Update): + update.status = ItemStatus.SUCCESS + update.wf_exec = mock.MagicMock() + # This is how the controller pulls the result from a successful execution + update.wf_exec.outputs.as_python_native.return_value = "hello" + +@mock.patch("flytekit.core.worker_queue.Controller.reconcile_one", side_effect=_mock_reconcile) +def test_controller(mock_reconcile): + print(f"ID mock_reconcile {id(mock_reconcile)}") + mock_reconcile.return_value = 123 -@mock.patch("flytekit.core.worker_queue.Controller.launch_and_start_watch") -def test_controller(mock_start): @task def t1() -> str: return "hello" @@ -21,29 +35,26 @@ def t1() -> str: ) c = Controller(remote, ss, tag="exec-id", root_tag="exec-id", exec_prefix="e-unit-test") - def _mock_start(wi: WorkItem, idx: int): - assert c.entries[wi.entity.name][idx] is wi - wi.wf_exec = mock.MagicMock() # just to pass the assert - wi.set_result("hello") - - mock_start.side_effect = _mock_start - async def fake_eager(): - loop = asyncio.get_running_loop() - f = c.add(loop, entity=t1, input_kwargs={}) + f = c.add(entity=t1, input_kwargs={}) res = await f assert res == "hello" loop_manager.run_sync(fake_eager) -@pytest.mark.asyncio -@mock.patch("flytekit.core.worker_queue.Controller") -async def test_controller_launch(mock_controller): +@mock.patch("flytekit.core.worker_queue.Controller._execute") +def test_controller_launch(mock_thread_target): @task def t2() -> str: return "hello" + def _mock_thread_target(*args, **kwargs): + print("in thread") + mock_thread_target.side_effect = _mock_thread_target + + wf_exec = mock.MagicMock() + def _mock_execute( entity, execution_name: str, @@ -57,53 +68,184 @@ def _mock_execute( assert execution_name.startswith("e-unit-test-t2-") assert envs == {'_F_EE_ROOT': 'exec-id'} print(entity, execution_name, inputs, version, image_config, options, envs) - wf_exec = mock.MagicMock() return wf_exec remote = mock.MagicMock() remote.execute.side_effect = _mock_execute - mock_controller.informer.watch.return_value = True - loop = asyncio.get_running_loop() - fut = loop.create_future() - wi = WorkItem(t2, input_kwargs={}, fut=fut) + wi = WorkItem(t2, input_kwargs={}) + + ss = SerializationSettings( + image_config=ImageConfig.auto_default_image(), + version="123", + ) + c = Controller(remote, ss, tag="exec-id", root_tag="exec-id", exec_prefix="e-unit-test") + + response_wf_exec = c.launch_execution(wi, 0) + assert response_wf_exec is wf_exec + + +@pytest.mark.asyncio +@mock.patch("flytekit.core.worker_queue.Controller.reconcile_one") +async def test_controller_update_cycle(mock_reconcile_one): + """ Test the whole update cycle end to end """ + @task + def t1() -> str: + return "hello" + remote = mock.MagicMock() ss = SerializationSettings( image_config=ImageConfig.auto_default_image(), + version="123", ) c = Controller(remote, ss, tag="exec-id", root_tag="exec-id", exec_prefix="e-unit-test") - c.launch_and_start_watch(wi, 0) - assert wi.error is None + def _mock_reconcile_one(update: Update): + print(f"in reconcile {update}") + update.status = ItemStatus.SUCCESS + update.wf_exec = mock.MagicMock() + update.wf_exec.outputs.as_python_native.return_value = "hello" + + mock_reconcile_one.side_effect = _mock_reconcile_one - wi.result = 5 - c.launch_and_start_watch(wi, 0) - # Function shouldn't be called if item already has a result - with pytest.raises(AssertionError): - await fut + add_coro = c.add(t1, input_kwargs={}) + res = await add_coro + assert res == "hello" @pytest.mark.asyncio -async def test_wi(): +@mock.patch("flytekit.core.worker_queue.Controller._execute") +async def test_controller_update_cycle_get_items(mock_thread_target): + """ Test just getting items to update """ + def _mock_thread_target(*args, **kwargs): + print("in thread") + mock_thread_target.side_effect = _mock_thread_target + @task def t1() -> str: return "hello" - loop = asyncio.get_running_loop() - fut = loop.create_future() - wi = WorkItem(t1, input_kwargs={}, fut=fut) + wi = WorkItem(t1, input_kwargs={}) + wi2 = WorkItem(t1, input_kwargs={}) - with pytest.raises(AssertionError): - wi.set_result("hello") + remote = mock.MagicMock() + ss = SerializationSettings( + image_config=ImageConfig.auto_default_image(), + version="123", + ) + c = Controller(remote, ss, tag="exec-id", root_tag="exec-id", exec_prefix="e-unit-test") + + c.entries["t1"] = [wi, wi2] + updates = c._get_update_items() + assert len(updates) == 2 + update_items = iter(updates.items()) + uuid, update = next(update_items) + assert uuid + assert update.work_item is wi + assert update.idx == 0 + assert update.status is None + assert update.wf_exec is None + assert update.error is None + + uuid_2, update_2 = next(update_items) + assert uuid != uuid_2 + assert update_2.idx == 1 + + +@pytest.mark.asyncio +@mock.patch("flytekit.core.worker_queue.Controller._execute") +async def test_controller_update_cycle_apply_updates(mock_thread_target): + def _mock_thread_target(*args, **kwargs): + print("in thread") + mock_thread_target.side_effect = _mock_thread_target + + @task + def t1() -> str: + return "hello" - assert not wi.ready + wi = WorkItem(t1, input_kwargs={}) + wi2 = WorkItem(t1, input_kwargs={}) - wi.wf_exec = mock.MagicMock() - wi.set_result("hello") - assert wi.ready + remote = mock.MagicMock() + ss = SerializationSettings( + image_config=ImageConfig.auto_default_image(), + version="123", + ) + c = Controller(remote, ss, tag="exec-id", root_tag="exec-id", exec_prefix="e-unit-test") + + c.entries["t1"] = [wi, wi2] + + wf_exec_1 = mock.MagicMock() + wf_exec_1.outputs.as_python_native.return_value = "hello" + + wf_exec_2 = mock.MagicMock() + wf_exec_2.closure.error = Exception("closure error") + + update_items = { + wi.uuid: Update(wi, 0, ItemStatus.SUCCESS, wf_exec_1, None), + wi2.uuid: Update(wi2, 1, ItemStatus.FAILED, wf_exec_2, None), + } + + c._apply_updates(update_items) + assert c.entries["t1"][0].status == ItemStatus.SUCCESS + assert c.entries["t1"][0].result == "hello" + assert c.entries["t1"][1].status == ItemStatus.FAILED + # errors in the closure are cast to eager exceptions + assert isinstance(c.entries["t1"][1].error, EagerException) + + update_items_second = { + wi2.uuid: Update(wi2, 1, ItemStatus.FAILED, wf_exec_2, ValueError("test value error")), + } + + c._apply_updates(update_items_second) + assert c.entries["t1"][0].status == ItemStatus.SUCCESS + assert c.entries["t1"][0].result == "hello" + assert c.entries["t1"][1].status == ItemStatus.FAILED + # errors set on the update object itself imply issues with the local code and are returned as is. + assert isinstance(c.entries["t1"][1].error, ValueError) + + +def test_work_item_hashing_equality(): + from flytekit.remote import FlyteRemote, FlyteWorkflowExecution + remote = FlyteRemote.for_sandbox(default_project="p", domain="d") + + e_spec = ExecutionSpec( + identifier.Identifier(identifier.ResourceType.LAUNCH_PLAN, "project", "domain", "name", "version"), + ExecutionMetadata(ExecutionMetadata.ExecutionMode.MANUAL, "tester", 1), + notifications=NotificationList( + [ + common_models.Notification( + [execution.WorkflowExecutionPhase.ABORTED], + pager_duty=common_models.PagerDutyNotification(recipients_email=["a", "b", "c"]), + ) + ] + ), + raw_output_data_config=common_models.RawOutputDataConfig(output_location_prefix="raw_output"), + max_parallelism=100, + ) + + test_datetime = datetime.datetime(year=2022, month=1, day=1, tzinfo=datetime.timezone.utc) + test_timedelta = datetime.timedelta(seconds=10) + abort_metadata = AbortMetadata(cause="cause", principal="testuser") + + e_closure = ExecutionClosure( + phase=execution.WorkflowExecutionPhase.SUCCEEDED, + started_at=test_datetime, + duration=test_timedelta, + abort_metadata=abort_metadata, + ) + + e_id = identifier.WorkflowExecutionIdentifier("project", "domain", "exec-name") + + ex = Execution(id=e_id, spec=e_spec, closure=e_closure) + + fwex = FlyteWorkflowExecution.promote_from_model(ex, remote) + + @task + def t1() -> str: + return "hello" - fut2 = loop.create_future() - wi = WorkItem(t1, input_kwargs={}, fut=fut2) - wi.set_error(ValueError("hello")) - with pytest.raises(ValueError): - await fut2 + wi1 = WorkItem(entity=t1, wf_exec=fwex, input_kwargs={}) + wi2 = WorkItem(entity=t1, wf_exec=fwex, input_kwargs={}) + wi2.uuid = wi1.uuid + assert wi1 == wi2 From abf0d411e57c942cb4ed708feb9383f0aa4a6be6 Mon Sep 17 00:00:00 2001 From: Paul Dittamo <37558497+pvditt@users.noreply.github.com> Date: Sat, 18 Jan 2025 07:56:16 -0800 Subject: [PATCH 08/27] Support with_overrides setting metadata for map_task subnode instead of parent node (#2982) * test Signed-off-by: Paul Dittamo * add support for with_overrides for map tasks Signed-off-by: Paul Dittamo * expand unit test Signed-off-by: Paul Dittamo * cleanup Signed-off-by: Paul Dittamo --------- Signed-off-by: Paul Dittamo --- flytekit/core/array_node_map_task.py | 12 +-- flytekit/core/node.py | 84 ++++++++++++------- flytekit/tools/translator.py | 2 +- .../unit/core/test_array_node_map_task.py | 58 +++++++++++-- 4 files changed, 110 insertions(+), 46 deletions(-) diff --git a/flytekit/core/array_node_map_task.py b/flytekit/core/array_node_map_task.py index 87150c47c1..05690e175b 100644 --- a/flytekit/core/array_node_map_task.py +++ b/flytekit/core/array_node_map_task.py @@ -128,6 +128,9 @@ def __init__( **kwargs, ) + self.sub_node_metadata: NodeMetadata = super().construct_node_metadata() + self.sub_node_metadata._name = self.name + @property def name(self) -> str: return self._name @@ -137,16 +140,13 @@ def python_interface(self): return self._collection_interface def construct_node_metadata(self) -> NodeMetadata: - # TODO: add support for other Flyte entities + """ + This returns metadata for the parent ArrayNode, not the sub-node getting mapped over + """ return NodeMetadata( name=self.name, ) - def construct_sub_node_metadata(self) -> NodeMetadata: - nm = super().construct_node_metadata() - nm._name = self.name - return nm - @property def min_success_ratio(self) -> Optional[float]: return self._min_success_ratio diff --git a/flytekit/core/node.py b/flytekit/core/node.py index ea089c6fd3..61ae41c060 100644 --- a/flytekit/core/node.py +++ b/flytekit/core/node.py @@ -124,6 +124,57 @@ def run_entity(self) -> Any: def metadata(self) -> _workflow_model.NodeMetadata: return self._metadata + def _override_node_metadata( + self, + name, + timeout: Optional[Union[int, datetime.timedelta]] = None, + retries: Optional[int] = None, + interruptible: typing.Optional[bool] = None, + cache: typing.Optional[bool] = None, + cache_version: typing.Optional[str] = None, + cache_serialize: typing.Optional[bool] = None, + ): + from flytekit.core.array_node_map_task import ArrayNodeMapTask + + if isinstance(self.flyte_entity, ArrayNodeMapTask): + # override the sub-node's metadata + node_metadata = self.flyte_entity.sub_node_metadata + else: + node_metadata = self._metadata + + if timeout is None: + node_metadata._timeout = datetime.timedelta() + elif isinstance(timeout, int): + node_metadata._timeout = datetime.timedelta(seconds=timeout) + elif isinstance(timeout, datetime.timedelta): + node_metadata._timeout = timeout + else: + raise ValueError("timeout should be duration represented as either a datetime.timedelta or int seconds") + if retries is not None: + assert_not_promise(retries, "retries") + node_metadata._retries = ( + _literal_models.RetryStrategy(0) if retries is None else _literal_models.RetryStrategy(retries) + ) + + if interruptible is not None: + assert_not_promise(interruptible, "interruptible") + node_metadata._interruptible = interruptible + + if name is not None: + node_metadata._name = name + + if cache is not None: + assert_not_promise(cache, "cache") + node_metadata._cacheable = cache + + if cache_version is not None: + assert_not_promise(cache_version, "cache_version") + node_metadata._cache_version = cache_version + + if cache_serialize is not None: + assert_not_promise(cache_serialize, "cache_serialize") + node_metadata._cache_serializable = cache_serialize + def with_overrides( self, node_name: Optional[str] = None, @@ -174,27 +225,6 @@ def with_overrides( assert_no_promises_in_resources(resources) self._resources = resources - if timeout is None: - self._metadata._timeout = datetime.timedelta() - elif isinstance(timeout, int): - self._metadata._timeout = datetime.timedelta(seconds=timeout) - elif isinstance(timeout, datetime.timedelta): - self._metadata._timeout = timeout - else: - raise ValueError("timeout should be duration represented as either a datetime.timedelta or int seconds") - if retries is not None: - assert_not_promise(retries, "retries") - self._metadata._retries = ( - _literal_models.RetryStrategy(0) if retries is None else _literal_models.RetryStrategy(retries) - ) - - if interruptible is not None: - assert_not_promise(interruptible, "interruptible") - self._metadata._interruptible = interruptible - - if name is not None: - self._metadata._name = name - if task_config is not None: logger.warning("This override is beta. We may want to revisit this in the future.") if not isinstance(task_config, type(self.run_entity._task_config)): @@ -209,17 +239,7 @@ def with_overrides( assert_not_promise(accelerator, "accelerator") self._extended_resources = tasks_pb2.ExtendedResources(gpu_accelerator=accelerator.to_flyte_idl()) - if cache is not None: - assert_not_promise(cache, "cache") - self._metadata._cacheable = cache - - if cache_version is not None: - assert_not_promise(cache_version, "cache_version") - self._metadata._cache_version = cache_version - - if cache_serialize is not None: - assert_not_promise(cache_serialize, "cache_serialize") - self._metadata._cache_serializable = cache_serialize + self._override_node_metadata(name, timeout, retries, interruptible, cache, cache_version, cache_serialize) return self diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index ee905a4218..e74f4c1c71 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -624,7 +624,7 @@ def get_serializable_array_node_map_task( ) node = workflow_model.Node( id=entity.name, - metadata=entity.construct_sub_node_metadata(), + metadata=entity.sub_node_metadata, inputs=node.bindings, upstream_node_ids=[], output_aliases=[], diff --git a/tests/flytekit/unit/core/test_array_node_map_task.py b/tests/flytekit/unit/core/test_array_node_map_task.py index ed1fc7fdd0..97693940e0 100644 --- a/tests/flytekit/unit/core/test_array_node_map_task.py +++ b/tests/flytekit/unit/core/test_array_node_map_task.py @@ -9,7 +9,7 @@ import pytest from flyteidl.core import workflow_pb2 as _core_workflow -from flytekit import dynamic, map_task, task, workflow, eager, PythonFunctionTask +from flytekit import dynamic, map_task, task, workflow, eager, PythonFunctionTask, Resources from flytekit.configuration import FastSerializationSettings, Image, ImageConfig, SerializationSettings from flytekit.core import context_manager from flytekit.core.array_node_map_task import ArrayNodeMapTask, ArrayNodeMapTaskResolver @@ -21,6 +21,7 @@ LiteralMap, LiteralOffloadedMetadata, ) +from flytekit.models.task import Resources as _resources_models from flytekit.tools.translator import get_serializable from flytekit.types.directory import FlyteDirectory @@ -349,16 +350,59 @@ def my_wf1() -> typing.List[typing.Optional[int]]: assert my_wf1() == [1, None, 3, 4] -def test_map_task_override(serialization_settings): - @task - def my_mappable_task(a: int) -> typing.Optional[str]: - return str(a) +@task +def my_mappable_task(a: int) -> typing.Optional[str]: + return str(a) + + +@task( + container_image="original-image", + timeout=timedelta(seconds=10), + interruptible=False, + retries=10, + cache=True, + cache_version="original-version", + requests=Resources(cpu=1) +) +def my_mappable_task_1(a: int) -> typing.Optional[str]: + return str(a) + + +@pytest.mark.parametrize( + "task_func", + [my_mappable_task, my_mappable_task_1] +) +def test_map_task_override(serialization_settings, task_func): + array_node_map_task = map_task(task_func) @workflow def wf(x: typing.List[int]): - map_task(my_mappable_task)(a=x).with_overrides(container_image="random:image") + array_node_map_task(a=x).with_overrides( + container_image="new-image", + timeout=timedelta(seconds=20), + interruptible=True, + retries=5, + cache=True, + cache_version="new-version", + requests=Resources(cpu=2) + ) + + assert wf.nodes[0]._container_image == "new-image" + + od = OrderedDict() + wf_spec = get_serializable(od, serialization_settings, wf) - assert wf.nodes[0]._container_image == "random:image" + array_node = wf_spec.template.nodes[0] + assert array_node.metadata.timeout == timedelta() + sub_node_spec = array_node.array_node.node + assert sub_node_spec.metadata.timeout == timedelta(seconds=20) + assert sub_node_spec.metadata.interruptible + assert sub_node_spec.metadata.retries.retries == 5 + assert sub_node_spec.metadata.cacheable + assert sub_node_spec.metadata.cache_version == "new-version" + assert sub_node_spec.target.overrides.resources.requests == [ + _resources_models.ResourceEntry(_resources_models.ResourceName.CPU, "2") + ] def test_serialization_metadata(serialization_settings): From 665c44df3dae88d33f5b32ac3d5d25711de899ed Mon Sep 17 00:00:00 2001 From: V <0426vincent@gmail.com> Date: Sat, 18 Jan 2025 08:02:41 -0800 Subject: [PATCH 09/27] fix: remove duplication log when execute (#3052) Signed-off-by: Vincent <0426vincent@gmail.com> --- flytekit/clis/sdk_in_container/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flytekit/clis/sdk_in_container/run.py b/flytekit/clis/sdk_in_container/run.py index 4c1da5ccf3..44141a5cc1 100644 --- a/flytekit/clis/sdk_in_container/run.py +++ b/flytekit/clis/sdk_in_container/run.py @@ -544,8 +544,9 @@ def run_remote( if run_level_params.wait_execution: msg += " Waiting to complete..." p = Progress(TimeElapsedColumn(), TextColumn(msg), transient=True) - t = p.add_task("exec") + t = p.add_task("exec", visible=False) with p: + p.update(t, visible=True) p.start_task(t) execution = remote.execute( entity, From 3260ddfea32b0a772cf002da643a02299be1021a Mon Sep 17 00:00:00 2001 From: "Fabio M. Graetz, Ph.D." Date: Sat, 18 Jan 2025 17:32:59 +0100 Subject: [PATCH 10/27] Fix: Always propagate pytorch task worker process exception timestamp to task exception (#3057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: Always propagate pytorch task worker process exception timestamp to task exception Signed-off-by: Fabio Grätz * Fix exist recoverable error test Signed-off-by: Fabio Grätz --------- Signed-off-by: Fabio Grätz Co-authored-by: Fabio Grätz --- flytekit/exceptions/user.py | 5 ++- .../flytekitplugins/kfpytorch/task.py | 4 +- .../tests/test_elastic_task.py | 38 ++++++++++++++++++- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/flytekit/exceptions/user.py b/flytekit/exceptions/user.py index af4dbf63c6..12cec206fc 100644 --- a/flytekit/exceptions/user.py +++ b/flytekit/exceptions/user.py @@ -15,14 +15,15 @@ class FlyteUserException(_FlyteException): class FlyteUserRuntimeException(_FlyteException): _ERROR_CODE = "USER:RuntimeError" - def __init__(self, exc_value: Exception): + def __init__(self, exc_value: Exception, timestamp: typing.Optional[float] = None): """ FlyteUserRuntimeException is thrown when a user code raises an exception. :param exc_value: The exception that was raised from user code. + :param timestamp: The timestamp as fractional seconds since epoch when the exception was raised. """ self._exc_value = exc_value - super().__init__(str(exc_value)) + super().__init__(str(exc_value), timestamp=timestamp) @property def value(self): diff --git a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py index a951bea0a5..4bbcb814a4 100644 --- a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py +++ b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py @@ -18,7 +18,7 @@ from flytekit.core.context_manager import FlyteContextManager, OutputMetadata from flytekit.core.pod_template import PodTemplate from flytekit.core.resources import convert_resources_to_resource_model -from flytekit.exceptions.user import FlyteRecoverableException +from flytekit.exceptions.user import FlyteRecoverableException, FlyteUserRuntimeException from flytekit.extend import IgnoreOutputs, TaskPlugins from flytekit.loggers import logger @@ -475,7 +475,7 @@ def fn_partial(): # the automatically assigned timestamp based on exception creation time raise FlyteRecoverableException(e.format_msg(), timestamp=first_failure.timestamp) else: - raise RuntimeError(e.format_msg()) + raise FlyteUserRuntimeException(e, timestamp=first_failure.timestamp) except SignalException as e: logger.exception(f"Elastic launch agent process terminating: {e}") raise IgnoreOutputs() diff --git a/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py b/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py index faadc1019f..f8742d1fe9 100644 --- a/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py +++ b/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py @@ -17,7 +17,7 @@ from flytekit import task, workflow from flytekit.core.context_manager import FlyteContext, FlyteContextManager, ExecutionState, ExecutionParameters, OutputMetadataTracker from flytekit.configuration import SerializationSettings -from flytekit.exceptions.user import FlyteRecoverableException +from flytekit.exceptions.user import FlyteRecoverableException, FlyteUserRuntimeException @pytest.fixture(autouse=True, scope="function") def restore_env(): @@ -223,7 +223,7 @@ def wf(recoverable: bool): with pytest.raises(FlyteRecoverableException): wf(recoverable=recoverable) else: - with pytest.raises(RuntimeError): + with pytest.raises(FlyteUserRuntimeException): wf(recoverable=recoverable) @@ -276,3 +276,37 @@ def test_task_omp_set(): assert os.environ["OMP_NUM_THREADS"] == "42" test_task_omp_set() + + +def test_exception_timestamp() -> None: + """Test that the timestamp of the worker process exception is propagated to the task exception.""" + @task( + task_config=Elastic( + nnodes=1, + nproc_per_node=2, + ) + ) + def test_task(): + raise Exception("Test exception") + + with pytest.raises(Exception) as e: + test_task() + + assert e.value.timestamp is not None + + +def test_recoverable_exception_timestamp() -> None: + """Test that the timestamp of the worker process exception is propagated to the task exception.""" + @task( + task_config=Elastic( + nnodes=1, + nproc_per_node=2, + ) + ) + def test_task(): + raise FlyteRecoverableException("Recoverable test exception") + + with pytest.raises(Exception) as e: + test_task() + + assert e.value.timestamp is not None From fb5fd3a7328ec48e8588b5b3c2a04ac37d21020e Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Tue, 21 Jan 2025 14:45:15 -0800 Subject: [PATCH 11/27] Eager deck name (#3072) Signed-off-by: Yee Hing Tong --- flytekit/core/python_function_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 5c7ad290aa..419da6e719 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -601,7 +601,7 @@ async def run_with_backend(self, **kwargs): base_error = ee html = cast(Controller, ctx.worker_queue).render_html() - Deck("eager workflow", html) + Deck("Eager Executions", html) if base_error: # now have to fail this eager task, because we don't want it to show up as succeeded. From a86048a094ee0d474c11a641a33d9288ff5e4bf8 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:59:12 -0500 Subject: [PATCH 12/27] Allow user-defined dataclass type transformer (again) (#3075) * Allow for user-defined dataclass type tranformers Signed-off-by: Eduardo Apolinario * Finish comment and remote user-defined dataclass transformer from registry Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 13 ++++++-- tests/flytekit/unit/core/test_type_engine.py | 35 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 2b4619540d..026921a205 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -1222,9 +1222,6 @@ def _get_transformer(cls, python_type: Type) -> Optional[TypeTransformer[T]]: if python_type in cls._REGISTRY: return cls._REGISTRY[python_type] - if dataclasses.is_dataclass(python_type): - return cls._DATACLASS_TRANSFORMER - return None @classmethod @@ -1243,6 +1240,16 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: if v is not None: return v + # flytekit's dataclass type transformer is left for last to give users a chance to register a type transformer + # to handle dataclass-like objects as part of the mro evaluation. + # + # N.B.: keep in mind that there are no compatibility guarantees between these user-defined dataclass transformers + # and the flytekit one. This incompatibility is *not* a new behavior introduced by the recent type engine + # refactor (https://github.com/flyteorg/flytekit/pull/2815), but it is worth calling out explicitly as a known + # limitation nonetheless. + if dataclasses.is_dataclass(python_type): + return cls._DATACLASS_TRANSFORMER + display_pickle_warning(str(python_type)) from flytekit.types.pickle.pickle import FlytePickleTransformer diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 48f7e4b959..5f5686b3f9 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -3742,3 +3742,38 @@ def test_structured_dataset_mismatch(): with pytest.raises(TypeTransformerFailedError): TypeEngine.to_literal(FlyteContext.current_context(), df, StructuredDataset, TypeEngine.to_literal_type(StructuredDataset)) + + +def test_register_dataclass_override(): + """ + Test to confirm that a dataclass transformer can be overridden by a user defined transformer + """ + + # We register a type transformer for the top-level user-defined dataclass + @dataclass + class ParentDC: + ... + + @dataclass + class ChildDC(ParentDC): + ... + + class ParentDCTransformer(TypeTransformer[ParentDC]): + def __init__(self): + super().__init__("ParentDC Transformer", ParentDC) + + # Register a type transformer for the parent dataclass + TypeEngine.register(ParentDCTransformer()) + + # Confirm that the transformer for ChildDC is the same as the ParentDC + assert TypeEngine.get_transformer(ChildDC) == TypeEngine.get_transformer(ParentDC) + + # Confirm that the transformer for ChildDC is not flytekit's default dataclass transformer + @dataclass + class RegularDC: + ... + + assert TypeEngine.get_transformer(ChildDC) != TypeEngine.get_transformer(RegularDC) + assert TypeEngine.get_transformer(RegularDC) == TypeEngine._DATACLASS_TRANSFORMER + + del TypeEngine._REGISTRY[ParentDC] From 7e33491b7caa50cf28f8fbcf693a76aea006d547 Mon Sep 17 00:00:00 2001 From: "Han-Ru Chen (Future-Outlier)" Date: Wed, 22 Jan 2025 21:43:33 +0800 Subject: [PATCH 13/27] [flyteagent][CLI] Make agent prometheus port configurable (#3064) * [flyteagent][CLI] Make agent prometheus port configurable Signed-off-by: Future-Outlier * lint Signed-off-by: Future-Outlier * update promethus port comment from Eduardo Signed-off-by: Future-Outlier Co-authored-by: Eduardo Apolinario * lint Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier Co-authored-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/serve.py | 19 +++++++---- .../clis/sdk_in_container/test_serve.py | 32 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/flytekit/clis/sdk_in_container/test_serve.py diff --git a/flytekit/clis/sdk_in_container/serve.py b/flytekit/clis/sdk_in_container/serve.py index d6ceab54fc..3d0d880e78 100644 --- a/flytekit/clis/sdk_in_container/serve.py +++ b/flytekit/clis/sdk_in_container/serve.py @@ -29,6 +29,13 @@ def serve(ctx: click.Context): type=int, help="Grpc port for the agent service", ) +@click.option( + "--prometheus_port", + default="9090", + is_flag=False, + type=int, + help="Prometheus port for the agent service", +) @click.option( "--worker", default="10", @@ -45,20 +52,20 @@ def serve(ctx: click.Context): "for testing.", ) @click.pass_context -def agent(_: click.Context, port, worker, timeout): +def agent(_: click.Context, port, prometheus_port, worker, timeout): """ Start a grpc server for the agent service. """ import asyncio - asyncio.run(_start_grpc_server(port, worker, timeout)) + asyncio.run(_start_grpc_server(port, prometheus_port, worker, timeout)) -async def _start_grpc_server(port: int, worker: int, timeout: int): +async def _start_grpc_server(port: int, prometheus_port: int, worker: int, timeout: int): from flytekit.extend.backend.agent_service import AgentMetadataService, AsyncAgentService, SyncAgentService click.secho("🚀 Starting the agent service...") - _start_http_server() + _start_http_server(prometheus_port) print_agents_metadata() server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=worker)) @@ -73,12 +80,12 @@ async def _start_grpc_server(port: int, worker: int, timeout: int): await server.wait_for_termination(timeout) -def _start_http_server(): +def _start_http_server(prometheus_port: int): try: from prometheus_client import start_http_server click.secho("Starting up the server to expose the prometheus metrics...") - start_http_server(9090) + start_http_server(prometheus_port) except ImportError as e: click.secho(f"Failed to start the prometheus server with error {e}", fg="red") diff --git a/tests/flytekit/clis/sdk_in_container/test_serve.py b/tests/flytekit/clis/sdk_in_container/test_serve.py new file mode 100644 index 0000000000..a1eeab1ba6 --- /dev/null +++ b/tests/flytekit/clis/sdk_in_container/test_serve.py @@ -0,0 +1,32 @@ +import pytest +from click.testing import CliRunner +from unittest.mock import patch + +from flytekit.clis.sdk_in_container.serve import serve + +def test_agent_prometheus_port(): + runner = CliRunner() + test_port = 9100 + test_prometheus_port = 9200 + test_worker = 5 + test_timeout = 30 + + with patch('flytekit.clis.sdk_in_container.serve._start_grpc_server') as mock_start_grpc: + result = runner.invoke( + serve, + [ + 'agent', + '--port', str(test_port), + '--prometheus_port', str(test_prometheus_port), + '--worker', str(test_worker), + '--timeout', str(test_timeout) + ] + ) + + assert result.exit_code == 0, f"Command failed with output: {result.output}" + mock_start_grpc.assert_called_once_with( + test_port, + test_prometheus_port, + test_worker, + test_timeout + ) From 768cd1b2ad5c0262e4c85891f98acb37da1c357f Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Wed, 22 Jan 2025 14:47:17 -0500 Subject: [PATCH 14/27] Adds python_exec into ImageSpec (#3069) Signed-off-by: Thomas J. Fan --- flytekit/image_spec/default_builder.py | 92 +++++++++++++------ flytekit/image_spec/image_spec.py | 2 + .../flytekitplugins/envd/image_builder.py | 3 + .../flytekit-envd/tests/test_image_spec.py | 15 +++ .../core/image_spec/test_default_builder.py | 37 ++++++++ 5 files changed, 119 insertions(+), 30 deletions(-) diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py index 29583a4386..2105a49acc 100644 --- a/flytekit/image_spec/default_builder.py +++ b/flytekit/image_spec/default_builder.py @@ -8,7 +8,7 @@ from pathlib import Path from string import Template from subprocess import run -from typing import ClassVar, List +from typing import ClassVar, List, NamedTuple import click @@ -78,6 +78,16 @@ $APT_PACKAGES """) +MICROMAMBA_INSTALL_COMMAND_TEMPLATE = Template("""\ +RUN --mount=type=cache,sharing=locked,mode=0777,target=/opt/micromamba/pkgs,\ +id=micromamba \ + --mount=from=micromamba,source=/usr/bin/micromamba,target=/usr/bin/micromamba \ + micromamba config set use_lockfiles False && \ + micromamba create -n runtime --root-prefix /opt/micromamba \ + -c conda-forge $CONDA_CHANNELS \ + python=$PYTHON_VERSION $CONDA_PACKAGES +""") + DOCKER_FILE_TEMPLATE = Template("""\ #syntax=docker/dockerfile:1.5 FROM ghcr.io/astral-sh/uv:0.5.1 as uv @@ -94,20 +104,14 @@ RUN id -u flytekit || useradd --create-home --shell /bin/bash flytekit RUN chown -R flytekit /root && chown -R flytekit /home -RUN --mount=type=cache,sharing=locked,mode=0777,target=/opt/micromamba/pkgs,\ -id=micromamba \ - --mount=from=micromamba,source=/usr/bin/micromamba,target=/usr/bin/micromamba \ - micromamba config set use_lockfiles False && \ - micromamba create -n runtime --root-prefix /opt/micromamba \ - -c conda-forge $CONDA_CHANNELS \ - python=$PYTHON_VERSION $CONDA_PACKAGES +$INSTALL_PYTHON_TEMPLATE # Configure user space -ENV PATH="/opt/micromamba/envs/runtime/bin:$$PATH" \ +ENV PATH="$EXTRA_PATH:$$PATH" \ + UV_PYTHON=$PYTHON_EXEC \ UV_LINK_MODE=copy \ FLYTE_SDK_RICH_TRACEBACKS=0 \ SSL_CERT_DIR=/etc/ssl/certs \ - UV_PYTHON=/opt/micromamba/envs/runtime/bin/python \ $ENV $UV_PYTHON_INSTALL_COMMAND @@ -240,6 +244,50 @@ def prepare_python_install(image_spec: ImageSpec, tmp_dir: Path) -> str: return UV_PYTHON_INSTALL_COMMAND_TEMPLATE.substitute(PIP_INSTALL_ARGS=pip_install_args) +class _PythonInstallTemplate(NamedTuple): + python_exec: str + template: str + extra_path: str + + +def prepare_python_executable(image_spec: ImageSpec) -> _PythonInstallTemplate: + if image_spec.python_exec: + if image_spec.conda_channels: + raise ValueError("conda_channels is not supported with python_exec") + if image_spec.conda_packages: + raise ValueError("conda_packages is not supported with python_exec") + return _PythonInstallTemplate(python_exec=image_spec.python_exec, template="", extra_path="") + + conda_packages = image_spec.conda_packages or [] + conda_channels = image_spec.conda_channels or [] + + if conda_packages: + conda_packages_concat = " ".join(conda_packages) + else: + conda_packages_concat = "" + + if conda_channels: + conda_channels_concat = " ".join(f"-c {channel}" for channel in conda_channels) + else: + conda_channels_concat = "" + + if image_spec.python_version: + python_version = image_spec.python_version + else: + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + + template = MICROMAMBA_INSTALL_COMMAND_TEMPLATE.substitute( + PYTHON_VERSION=python_version, + CONDA_PACKAGES=conda_packages_concat, + CONDA_CHANNELS=conda_channels_concat, + ) + return _PythonInstallTemplate( + python_exec="/opt/micromamba/envs/runtime/bin/python", + template=template, + extra_path="/opt/micromamba/envs/runtime/bin", + ) + + def create_docker_context(image_spec: ImageSpec, tmp_dir: Path): """Populate tmp_dir with Dockerfile as specified by the `image_spec`.""" base_image = image_spec.base_image or "debian:bookworm-slim" @@ -300,23 +348,7 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path): else: copy_command_runtime = "" - conda_packages = image_spec.conda_packages or [] - conda_channels = image_spec.conda_channels or [] - - if conda_packages: - conda_packages_concat = " ".join(conda_packages) - else: - conda_packages_concat = "" - - if conda_channels: - conda_channels_concat = " ".join(f"-c {channel}" for channel in conda_channels) - else: - conda_channels_concat = "" - - if image_spec.python_version: - python_version = image_spec.python_version - else: - python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + python_install_template = prepare_python_executable(image_spec=image_spec) if image_spec.entrypoint is None: entrypoint = "" @@ -351,11 +383,11 @@ def create_docker_context(image_spec: ImageSpec, tmp_dir: Path): extra_copy_cmds = "" docker_content = DOCKER_FILE_TEMPLATE.substitute( - PYTHON_VERSION=python_version, UV_PYTHON_INSTALL_COMMAND=uv_python_install_command, - CONDA_PACKAGES=conda_packages_concat, - CONDA_CHANNELS=conda_channels_concat, APT_INSTALL_COMMAND=apt_install_command, + INSTALL_PYTHON_TEMPLATE=python_install_template.template, + EXTRA_PATH=python_install_template.extra_path, + PYTHON_EXEC=python_install_template.python_exec, BASE_IMAGE=base_image, ENV=env, COPY_COMMAND_RUNTIME=copy_command_runtime, diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 85920ffae3..1b9e6403e1 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -62,6 +62,7 @@ class ImageSpec: If the option is set by the user, then that option is of course used. copy: List of files/directories to copy to /root. e.g. ["src/file1.txt", "src/file2.txt"] + python_exec: Python executable to use for install packages """ name: str = "flytekit" @@ -87,6 +88,7 @@ class ImageSpec: tag_format: Optional[str] = None source_copy_mode: Optional[CopyFileDetection] = None copy: Optional[List[str]] = None + python_exec: Optional[str] = None def __post_init__(self): self.name = self.name.lower() diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index bc8ef7de68..3e160a8a0f 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -98,6 +98,9 @@ def _create_str_from_package_list(packages): def create_envd_config(image_spec: ImageSpec) -> str: + if image_spec.python_exec is not None: + raise ValueError("python_exec is not supported with the envd image builder") + base_image = DefaultImages.default_image() if image_spec.base_image is None else image_spec.base_image if image_spec.cuda: if image_spec.python_version is None: diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index e7b87cc1cc..b096d80da7 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -131,3 +131,18 @@ def build(): ) assert contents == expected_contents + + +def test_envd_failure_python_exec(): + base_image = "ghcr.io/flyteorg/flytekit:py3.11-1.14.4" + python_exec = "/usr/local/bin/python" + + image_spec = ImageSpec( + name="FLYTEKIT", + base_image=base_image, + python_exec=python_exec + ) + + msg = "python_exec is not supported with the envd image builder" + with pytest.raises(ValueError, match=msg): + EnvdImageSpecBuilder().build_image(image_spec) diff --git a/tests/flytekit/unit/core/image_spec/test_default_builder.py b/tests/flytekit/unit/core/image_spec/test_default_builder.py index 0abd3d9467..aadcac0a16 100644 --- a/tests/flytekit/unit/core/image_spec/test_default_builder.py +++ b/tests/flytekit/unit/core/image_spec/test_default_builder.py @@ -318,3 +318,40 @@ def test_create_poetry_lock(tmp_path): dockerfile_content = dockerfile_path.read_text() assert "poetry install --no-root" in dockerfile_content + + +def test_python_exec(tmp_path): + docker_context_path = tmp_path / "builder_root" + docker_context_path.mkdir() + base_image = "ghcr.io/flyteorg/flytekit:py3.11-1.14.4" + python_exec = "/usr/local/bin/python" + + image_spec = ImageSpec( + name="FLYTEKIT", + base_image=base_image, + python_exec=python_exec + ) + + create_docker_context(image_spec, docker_context_path) + + dockerfile_path = docker_context_path / "Dockerfile" + assert dockerfile_path.exists() + dockerfile_content = dockerfile_path.read_text() + + assert f"UV_PYTHON={python_exec}" in dockerfile_content + + +@pytest.mark.parametrize("key, value", [("conda_packages", ["ruff"]), ("conda_channels", ["bioconda"])]) +def test_python_exec_errors(tmp_path, key, value): + docker_context_path = tmp_path / "builder_root" + docker_context_path.mkdir() + + image_spec = ImageSpec( + name="FLYTEKIT", + base_image="ghcr.io/flyteorg/flytekit:py3.11-1.14.4", + python_exec="/usr/local/bin/python", + **{key: value} + ) + msg = f"{key} is not supported with python_exec" + with pytest.raises(ValueError, match=msg): + create_docker_context(image_spec, docker_context_path) From e9fa4fb6f0a29203fa39ae366b5b4803e1a392d5 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Wed, 22 Jan 2025 11:52:03 -0800 Subject: [PATCH 15/27] add a test (#3074) Signed-off-by: Yee Hing Tong --- tests/flytekit/unit/core/test_dataclass.py | 45 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/flytekit/unit/core/test_dataclass.py b/tests/flytekit/unit/core/test_dataclass.py index 4e098c254b..09a6d26f54 100644 --- a/tests/flytekit/unit/core/test_dataclass.py +++ b/tests/flytekit/unit/core/test_dataclass.py @@ -6,6 +6,7 @@ import sys import tempfile from dataclasses import dataclass, fields, field +from dataclasses_json import DataClassJsonMixin, dataclass_json from typing import List, Dict, Optional, Union, Any from typing_extensions import Annotated from flytekit.types.schema import FlyteSchema @@ -17,6 +18,7 @@ from flytekit.types.file import FlyteFile from flytekit.types.structured import StructuredDataset + @pytest.fixture def local_dummy_txt_file(): fd, path = tempfile.mkstemp(suffix=".txt") @@ -27,6 +29,7 @@ def local_dummy_txt_file(): finally: os.remove(path) + @pytest.fixture def local_dummy_directory(): temp_dir = tempfile.TemporaryDirectory() @@ -37,6 +40,7 @@ def local_dummy_directory(): finally: temp_dir.cleanup() + def test_dataclass(): @dataclass class AppParams(DataClassJsonMixin): @@ -66,6 +70,7 @@ class MyDC(DataClassJSONMixin): d = Annotated[MyDC, "tag"] DataclassTransformer().assert_type(d, MyDC(my_str="hi")) + def test_pure_dataclasses_with_python_types(): @dataclass class DC: @@ -330,6 +335,7 @@ class NestedFlyteTypes: assert isinstance(pv, NestedFlyteTypes) DataclassTransformer().assert_type(NestedFlyteTypes, pv) + ## For dataclasses json mixin, it's equal to use @dataclasses_json def test_dataclasses_json_mixin_with_python_types(): @dataclass @@ -402,7 +408,6 @@ class DCWithOptional(DataClassJsonMixin): ctx = FlyteContextManager.current_context() - o = DCWithOptional() lt = TypeEngine.to_literal_type(DCWithOptional) lv = TypeEngine.to_literal(ctx, o, DCWithOptional, lt) @@ -596,6 +601,7 @@ class NestedFlyteTypes(DataClassJsonMixin): assert isinstance(pv, NestedFlyteTypes) DataclassTransformer().assert_type(NestedFlyteTypes, pv) + # For mashumaro dataclasses mixins, it's equal to use @dataclasses only def test_mashumaro_dataclasses_json_mixin_with_python_types(): @dataclass @@ -650,6 +656,34 @@ def t2() -> DCWithOptional: DataclassTransformer().assert_type(DCWithOptional, dc2) +def test_ret_unions(): + @dataclass_json + @dataclass + class DC: + my_string: str + + @dataclass_json + @dataclass + class DCWithOptional: + my_float: float + + @task + def make_union(a: int) -> Union[DC, DCWithOptional]: + if a > 10: + return DC(my_string="hello") + else: + return DCWithOptional(my_float=3.14) + + @workflow + def make_union_wf(a: int) -> Union[DC, DCWithOptional]: + return make_union(a=a) + + dc = make_union_wf(a=15) + assert dc.my_string == "hello" + dc = make_union_wf(a=5) + assert dc.my_float == 3.14 + + def test_mashumaro_dataclasses_json_mixin_with_python_types_get_literal_type_and_to_python_value(): @dataclass class DC(DataClassJSONMixin): @@ -861,6 +895,7 @@ class NestedFlyteTypes(DataClassJSONMixin): assert isinstance(pv, NestedFlyteTypes) DataclassTransformer().assert_type(NestedFlyteTypes, pv) + def test_get_literal_type_data_class_json_fail_but_mashumaro_works(): @dataclass class FlyteTypesWithDataClassJson(DataClassJsonMixin): @@ -885,6 +920,8 @@ class NestedFlyteTypesWithDataClassJson(DataClassJsonMixin): transformer = DataclassTransformer() lt = transformer.get_literal_type(NestedFlyteTypesWithDataClassJson) assert lt.metadata is not None + + @pytest.mark.skipif(sys.version_info < (3, 10), reason="Requires Python 3.10 or higher") def test_numpy_import_issue_from_flyte_schema_in_dataclass(): from dataclasses import dataclass @@ -916,6 +953,7 @@ def main_flyte_workflow(b: bool = False) -> bool: assert main_flyte_workflow(b=True) == True assert main_flyte_workflow(b=False) == False + def test_dataclass_union_primitive_types_and_enum(): class Status(Enum): PENDING = "pending" @@ -934,6 +972,7 @@ def my_task(dc: DC) -> DC: my_task(dc=DC()) + def test_dataclass_with_json_mixin_union_primitive_types_and_enum(): class Status(Enum): PENDING = "pending" @@ -952,6 +991,7 @@ def my_task(dc: DC) -> DC: my_task(dc=DC()) + def test_frozen_dataclass(): @dataclass(frozen=True) class FrozenDataclass: @@ -970,6 +1010,7 @@ def t1(dc: FrozenDataclass) -> (int, float, bool, str): assert c == True assert d == "hello" + def test_pure_frozen_dataclasses_with_python_types(): @dataclass(frozen=True) class DC: @@ -1022,6 +1063,7 @@ def t2() -> DCWithOptional: DataclassTransformer().assert_type(DCWithOptional, dc1) DataclassTransformer().assert_type(DCWithOptional, dc2) + def test_pure_frozen_dataclasses_with_flyte_types(local_dummy_txt_file, local_dummy_directory): @dataclass(frozen=True) class FlyteTypes: @@ -1119,6 +1161,7 @@ def empty_nested_dc_wf() -> NestedFlyteTypes: empty_nested_flyte_types = empty_nested_dc_wf() DataclassTransformer().assert_type(NestedFlyteTypes, empty_nested_flyte_types) + def test_dataclass_serialize_with_multiple_dataclass_union(): @dataclass class A(): From 8a6bbd05e5a2bff7d0cfab58f3cb6e2beac8dfe1 Mon Sep 17 00:00:00 2001 From: Shuying Liang <33474827+shuyingliang@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:16:58 -0800 Subject: [PATCH 16/27] Add the Flyte agent to provision and manage K8s (data) service for deep learning (GNN) use cases (#3004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Shuying Liang Signed-off-by: Future-Outlier Signed-off-by: JiaWei Jiang Signed-off-by: Yee Hing Tong Signed-off-by: Niels Bantilan Co-authored-by: Han-Ru Chen (Future-Outlier) Co-authored-by: Thomas J. Fan Co-authored-by: 江家瑋 <36886416+JiangJiaWei1103@users.noreply.github.com> Co-authored-by: Yee Hing Tong Co-authored-by: Niels Bantilan --- .github/workflows/pythonbuild.yml | 1 + .pre-commit-config.yaml | 7 +- Dockerfile.agent | 1 + .../source/plugins/k8sstatefuldataservice.rst | 12 + plugins/flytekit-k8sdataservice/README.md | 104 ++++++ .../dev-requirements.txt | 1 + .../k8sdataservice/__init__.py | 15 + .../flytekitplugins/k8sdataservice/agent.py | 98 ++++++ .../k8sdataservice/k8s/__init__.py | 0 .../k8sdataservice/k8s/kube_config.py | 20 ++ .../k8sdataservice/k8s/manager.py | 219 ++++++++++++ .../flytekitplugins/k8sdataservice/sensor.py | 67 ++++ .../flytekitplugins/k8sdataservice/task.py | 71 ++++ plugins/flytekit-k8sdataservice/setup.py | 38 +++ .../k8sdataservice/k8s/test_kube_config.py | 23 ++ .../tests/k8sdataservice/k8s/test_manager.py | 168 +++++++++ .../tests/k8sdataservice/test_agent.py | 319 ++++++++++++++++++ .../tests/k8sdataservice/test_sensor.py | 134 ++++++++ .../tests/k8sdataservice/test_task.py | 128 +++++++ .../k8sdataservice/utils/test_resources.py | 90 +++++ .../flytekit-k8sdataservice/utils/__init__.py | 0 .../flytekit-k8sdataservice/utils/infra.py | 9 + .../utils/resources.py | 24 ++ plugins/setup.py | 1 + .../integration/remote/test_remote.py | 9 + .../core/test_generice_idl_type_engine.py | 6 +- tests/flytekit/unit/core/test_type_engine.py | 6 +- 27 files changed, 1562 insertions(+), 9 deletions(-) create mode 100644 docs/source/plugins/k8sstatefuldataservice.rst create mode 100644 plugins/flytekit-k8sdataservice/README.md create mode 100644 plugins/flytekit-k8sdataservice/dev-requirements.txt create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/__init__.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/agent.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/__init__.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/kube_config.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/manager.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/sensor.py create mode 100644 plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/task.py create mode 100644 plugins/flytekit-k8sdataservice/setup.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_kube_config.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_manager.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_agent.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_sensor.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_task.py create mode 100644 plugins/flytekit-k8sdataservice/tests/k8sdataservice/utils/test_resources.py create mode 100644 plugins/flytekit-k8sdataservice/utils/__init__.py create mode 100644 plugins/flytekit-k8sdataservice/utils/infra.py create mode 100644 plugins/flytekit-k8sdataservice/utils/resources.py diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index b1ed59cead..d6f98fb771 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -303,6 +303,7 @@ jobs: - flytekit-huggingface - flytekit-identity-aware-proxy - flytekit-inference + - flytekit-k8sdataservice - flytekit-k8s-pod - flytekit-kf-mpi - flytekit-kf-pytorch diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f6999cf11..ea236cc92f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.6.9 + rev: v0.8.3 hooks: # Run the linter. - id: ruff @@ -26,5 +26,6 @@ repos: rev: v2.3.0 hooks: - id: codespell - additional_dependencies: - - tomli + args: + - --ignore-words-list=assertIn # Ignore 'assertIn' + additional_dependencies: [tomli] diff --git a/Dockerfile.agent b/Dockerfile.agent index 1715fde894..b95ed9da56 100644 --- a/Dockerfile.agent +++ b/Dockerfile.agent @@ -11,6 +11,7 @@ RUN apt-get update && apt-get install build-essential -y \ RUN uv pip install --system --no-cache-dir -U flytekit==$VERSION \ flytekitplugins-airflow==$VERSION \ flytekitplugins-bigquery==$VERSION \ + flytekitplugins-k8sdataservice==$VERSION \ flytekitplugins-openai==$VERSION \ flytekitplugins-snowflake==$VERSION \ flytekitplugins-awssagemaker==$VERSION \ diff --git a/docs/source/plugins/k8sstatefuldataservice.rst b/docs/source/plugins/k8sstatefuldataservice.rst new file mode 100644 index 0000000000..09abe42d29 --- /dev/null +++ b/docs/source/plugins/k8sstatefuldataservice.rst @@ -0,0 +1,12 @@ +.. k8sstatefuldataservice: + +################################################### +Kubernetes StatefulSet Data Service API reference +################################################### + +.. tags:: Integration, DeepLearning, MachineLearning, Kubernetes, GNN + +.. automodule:: flytekitplugins.k8sdataservice + :no-members: + :no-inherited-members: + :no-special-members: diff --git a/plugins/flytekit-k8sdataservice/README.md b/plugins/flytekit-k8sdataservice/README.md new file mode 100644 index 0000000000..6e776982c4 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/README.md @@ -0,0 +1,104 @@ +# K8s Stateful Service Plugin + +This plugin provides support for Kubernetes StatefulSet and Service integration, enabling seamless provisioning and coordination with any Kubernetes services or Flyte tasks. It is especially suited for deep learning use cases at scale, where distributed and parallelized data loading and caching across nodes are required. + +## Features +- **Predictable and Reliable Endpoints**: The service creates consistent endpoints, facilitating communication between services or tasks within the same Kubernetes cluster. +- **Reusable Across Runs**: Service tasks can persist across task runs, ensuring consistency. Alternatively, a cleanup sensor can release cluster resources when they are no longer needed. +- **Conventional Pod Naming**: Pods in the StatefulSet follow a conventional naming pattern. For instance, if the StatefulSet name is `foo` and replicas are set to 2, the pod endpoints will be `foo-0.foo:1234` and `foo-1.foo:1234`. This simplifies endpoint construction for training or inference scripts. For example, gRPC endpoints can directly use `foo-0.foo:1234` and `foo-1.foo:1234`. + +## Installation + +Install the plugin via pip: + +```bash +pip install flytekitplugins-k8sdataservice +``` + +## Usage + +Below is an example demonstrating how to provision and run a service in Kubernetes, making it reachable within the cluster. + +**Note**: Utility functions are available to generate unique service names that can be reused across training or inference scripts. + +### Example Usage + +#### Provisioning a Data Service +```python +from flytekitplugins.k8sdataservice import DataServiceConfig, DataServiceTask, CleanupSensor +from utils.infra import gen_infra_name +from flytekit import kwtypes, Resources, task, workflow + +# Generate a unique infrastructure name +name = gen_infra_name() + +def k8s_data_service(): + gnn_config = DataServiceConfig( + Name=name, + Requests=Resources(cpu='1', mem='1Gi'), + Limits=Resources(cpu='2', mem='2Gi'), + Replicas=1, + Image="busybox:latest", + Command=[ + "bash", + "-c", + "echo Hello Flyte K8s Stateful Service! && sleep 3600" + ], + ) + + gnn_task = DataServiceTask( + name="K8s Stateful Data Service", + inputs=kwtypes(ds=str), + task_config=gnn_config, + ) + return gnn_task + +# Define a cleanup sensor +gnn_sensor = CleanupSensor(name="Cleanup") + +# Define a workflow to test the data service +@workflow +def test_dataservice_wf(name: str): + k8s_data_service()(ds="OSS Flyte K8s Data Service Demo") \ + >> gnn_sensor( + release_name=name, + cleanup_data_service=True, + ) + +if __name__ == "__main__": + out = test_dataservice_wf(name="example") + print(f"Running test_dataservice_wf() {out}") +``` + +#### Accessing the Data Service +Other tasks or services that need to access the data service can do so in multiple ways. For example, using environment variables: + +```python +from kubernetes.client import V1PodSpec, V1Container, V1EnvVar + +PRIMARY_CONTAINER_NAME = "primary" +FLYTE_POD_SPEC = V1PodSpec( + containers=[ + V1Container( + name=PRIMARY_CONTAINER_NAME, + env=[ + V1EnvVar(name="MY_DATASERVICES", value=f"{name}-0.{name}:40000 {name}-1.{name}:40000"), + ], + ) + ], +) + +task_config = MPIJob( + launcher=Launcher(replicas=1, pod_template=FLYTE_POD_SPEC), + worker=Worker(replicas=1, pod_template=FLYTE_POD_SPEC), +) + +@task(task_config=task_config) +def mpi_task() -> str: + return "your script uses the envs to communicate with the data service " +``` + +### Key Points +- The `DataServiceConfig` defines resource requests, limits, replicas, and the container image/command. +- The `CleanupSensor` ensures resources are cleaned up when required. +- The workflow connects the service provisioning and cleanup process for streamlined operations. diff --git a/plugins/flytekit-k8sdataservice/dev-requirements.txt b/plugins/flytekit-k8sdataservice/dev-requirements.txt new file mode 100644 index 0000000000..83f3480e59 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/dev-requirements.txt @@ -0,0 +1 @@ +kubernetes~=23.6.0 diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/__init__.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/__init__.py new file mode 100644 index 0000000000..f85aa90be0 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/__init__.py @@ -0,0 +1,15 @@ +""" +.. currentmodule:: flytekitplugins.k8sdataservice + +This package contains things that are useful when extending Flytekit. + +.. autosummary:: + :template: custom.rst + :toctree: generated/ + + DataServiceTask +""" + +from .agent import DataServiceAgent # noqa: F401 +from .sensor import CleanupSensor # noqa: F401 +from .task import DataServiceConfig, DataServiceTask # noqa: F401 diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/agent.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/agent.py new file mode 100644 index 0000000000..199db535bd --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/agent.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import Optional + +from flyteidl.core.execution_pb2 import TaskExecution +from flytekitplugins.k8sdataservice.k8s.manager import K8sManager +from flytekitplugins.k8sdataservice.task import DataServiceConfig + +from flytekit import logger +from flytekit.extend.backend.base_agent import AgentRegistry, AsyncAgentBase, Resource, ResourceMeta +from flytekit.models.literals import LiteralMap +from flytekit.models.task import TaskTemplate + + +@dataclass +class DataServiceMetadata(ResourceMeta): + dataservice_config: DataServiceConfig + name: str + + +class DataServiceAgent(AsyncAgentBase): + name = "K8s DataService Async Agent" + + def __init__(self): + self.k8s_manager = K8sManager() + super().__init__(task_type_name="dataservicetask", metadata_type=DataServiceMetadata) + self.config = None + + def create( + self, task_template: TaskTemplate, output_prefix: str, inputs: Optional[LiteralMap] = None, **kwargs + ) -> DataServiceMetadata: + graph_engine_config = task_template.custom + self.k8s_manager.set_configs(graph_engine_config) + logger.info(f"Loaded agent config file {self.config}") + existing_release_name = graph_engine_config.get("ExistingReleaseName", None) + logger.info(f"The existing data service release name is {existing_release_name}") + + name = "" + if existing_release_name is None or existing_release_name == "": + logger.info("Creating K8s data service resources...") + name = self.k8s_manager.create_data_service() + logger.info(f'Data service {name} with image {graph_engine_config["Image"]} completed') + else: + name = existing_release_name + logger.info(f"User configs to use the existing data service release name: {name}.") + + dataservice_config = DataServiceConfig( + Name=graph_engine_config.get("Name", None), + Image=graph_engine_config["Image"], + Command=graph_engine_config["Command"], + Cluster=graph_engine_config["Cluster"], + ExistingReleaseName=graph_engine_config.get("ExistingReleaseName", None), + ) + metadata = DataServiceMetadata( + dataservice_config=dataservice_config, + name=name, + ) + logger.info(f"Created DataService metadata {metadata}") + return metadata + + def get(self, resource_meta: DataServiceMetadata) -> Resource: + logger.info("K8s Data Service get is called") + data = resource_meta.dataservice_config + data_dict = data.__dict__ if isinstance(data, DataServiceConfig) else data + logger.info(f"The data_dict is {data_dict}") + self.k8s_manager.set_configs(data_dict) + name = data.Name + logger.info(f"Get the stateful set name {name}") + + k8s_status = self.k8s_manager.check_stateful_set_status(name) + flyte_state = None + if k8s_status in ["failed", "timeout", "timedout", "canceled", "skipped", "internal_error"]: + flyte_state = TaskExecution.FAILED + elif k8s_status in ["done", "succeeded", "success"]: + flyte_state = TaskExecution.SUCCEEDED + elif k8s_status in ["running", "terminating", "pending"]: + flyte_state = TaskExecution.RUNNING + else: + logger.error(f"Unrecognized state: {k8s_status}") + outputs = { + "data_service_name": name, + } + # TODO: Add logs for StatefulSet. + return Resource(phase=flyte_state, outputs=outputs) + + def delete(self, resource_meta: DataServiceMetadata): + logger.info("DataService delete is called") + data = resource_meta.dataservice_config + + data_dict = data.__dict__ if isinstance(data, DataServiceConfig) else data + self.k8s_manager.set_configs(data_dict) + + name = resource_meta.name + logger.info(f"To delete the DataService (e.g., StatefulSet and Service) with name {name}") + self.k8s_manager.delete_stateful_set(name) + self.k8s_manager.delete_service(name) + + +AgentRegistry.register(DataServiceAgent()) diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/__init__.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/kube_config.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/kube_config.py new file mode 100644 index 0000000000..97d1297c7f --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/kube_config.py @@ -0,0 +1,20 @@ +from kubernetes import config + +from flytekit import logger + + +class KubeConfig: + def __init__(self): + pass + + def load_kube_config(self) -> None: + """Load the kubernetes config based on fabric details prior to K8s client usage + + :params target_fabric: fabric on which we are loading configs + """ + try: + logger.info("Attempting to load in-cluster configuration.") + config.load_incluster_config() # This will use the service account credentials + logger.info("Successfully loaded in-cluster configuration using the agent service account.") + except config.ConfigException as e: + logger.warning(f"Failed to load in-cluster configuration. {e}") diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/manager.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/manager.py new file mode 100644 index 0000000000..a02fb06a3c --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/k8s/manager.py @@ -0,0 +1,219 @@ +import uuid + +from flytekitplugins.k8sdataservice.k8s.kube_config import KubeConfig +from kubernetes import client +from kubernetes.client.rest import ApiException +from utils.resources import cleanup_resources, convert_flyte_to_k8s_fields + +from flytekit import logger + +APPNAME = "service-name" +DEFAULT_RESOURCES = client.V1ResourceRequirements( + requests={"cpu": "2", "memory": "10G"}, limits={"cpu": "6", "memory": "16G"} +) + + +class K8sManager: + def __init__(self): + self.config = KubeConfig() + self.config.load_kube_config() + self.apps_v1_api = client.AppsV1Api() + self.core_v1_api = client.CoreV1Api() + + def set_configs(self, data_service_config): + self.data_service_config = data_service_config + self.labels = {} + self.namespace = "flyte" + self.name = None + self.name = data_service_config.get("Name", None) + if self.name is None: + self.name = f"k8s-dataservice-{uuid.uuid4().hex[:8]}" + + def create_data_service(self) -> str: + svc_name = self.create_service() + logger.info(f"Created service: {svc_name}") + stateful_set_obj = self.create_stateful_set_object() + name = self.create_stateful_set(stateful_set_obj) + return name + + def create_stateful_set(self, stateful_set_object) -> str: + api_response = None + try: + api_response = self.apps_v1_api.create_namespaced_stateful_set( + namespace=self.namespace, body=stateful_set_object + ) + logger.info(f"Created statefulset in K8s API server: {api_response}") + except ApiException as e: + logger.error(f"Exception when calling AppsV1Api->create_namespaced_stateful_set: {e}\n") + raise + return api_response.metadata.name + + def create_stateful_set_object(self): + container = self._create_container() + template = self._create_pod_template(container) + spec = self._create_stateful_set_spec(template) + return client.V1StatefulSet( + api_version="apps/v1", + kind="StatefulSet", + metadata=client.V1ObjectMeta( + labels=self.labels, + name=self.name, + annotations={}, + ), + spec=spec, + ) + + def _create_container(self): + ss_replicas = self.data_service_config.get("Replicas", 1) + port = self.data_service_config.get("Port", 40000) + ss_env = [ + client.V1EnvVar(name="GE_BASE_PORT", value=str(port)), + client.V1EnvVar(name="GE_COUNT", value=str(int(ss_replicas))), + client.V1EnvVar(name="SERVER_PORT", value=str(port)), + ] + return client.V1Container( + name=self.name, + image=self.data_service_config["Image"], + image_pull_policy="IfNotPresent", + ports=[client.V1ContainerPort(container_port=port, name="graph-engine")], + command=self.data_service_config["Command"], + env=ss_env, + resources=self.get_resources(), + ) + + def _create_pod_template(self, container): + self.labels.update({"app.kubernetes.io/instance": self.name}) + return client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta( + labels=self.labels, + annotations={}, + ), + spec=client.V1PodSpec( + containers=[container], + security_context=client.V1PodSecurityContext( + fs_group=1001, + run_as_group=1001, + run_as_non_root=True, + run_as_user=1001, + ), + ), + ) + + def _create_stateful_set_spec(self, template): + ss_replicas = self.data_service_config.get("Replicas", 1) + return client.V1StatefulSetSpec( + replicas=int(ss_replicas), + selector=client.V1LabelSelector( + match_labels={"app.kubernetes.io/instance": self.name}, + ), + service_name=self.name, + template=template, + ) + + def create_service(self) -> str: + namespace = self.namespace + logger.info(f"creating a service at namespace {namespace} with name {self.name}") + port = self.data_service_config.get("Port", 40000) + self.labels.update({"app.kubernetes.io/instance": self.name, "app": APPNAME}) + body = client.V1Service( + api_version="v1", + kind="Service", + metadata=client.V1ObjectMeta( + name=self.name, + labels=self.labels, + namespace=namespace, + ), + spec=client.V1ServiceSpec( + selector={"app.kubernetes.io/instance": self.name}, + type="ClusterIP", + ports=[ + client.V1ServicePort( + port=port, + target_port=port, + name=self.name, + ) + ], + ), + ) + logger.info( + f"Service configuration in namespace {namespace} and name {self.name} is completed, posting request to K8s API server..." + ) + api_response = None + try: + api_response = self.core_v1_api.create_namespaced_service(namespace=namespace, body=body) + except ApiException as e: + logger.error(f"Exception when calling CoreV1Api->create_namespaced_service: {e}") + raise e + # This will not happen in K8s API, but in case. + if api_response is None or not hasattr(api_response, "metadata") or not hasattr(api_response.metadata, "name"): + raise ValueError("Invalid response from Kubernetes API - missing metadata or name") + return api_response.metadata.name + + def check_stateful_set_status(self, name) -> str: + try: + stateful_set = self.apps_v1_api.read_namespaced_stateful_set(name=name, namespace=self.namespace) + status = stateful_set.status + logger.info(f"StatefulSet status: {status}") + conditions = status.conditions if status and status.conditions else [] + logger.info(f"StatefulSet conditions: {conditions}") + + if status.replicas == 0: + logger.info( + f"StatefulSet {name} is pending. replicas: {status.replicas}, available: {status.available_replicas }" + ) + return "pending" + + if status.replicas > 0 and ( + status.replicas == status.available_replicas or status.replicas == status.ready_replicas + ): + logger.info( + f"StatefulSet {name} has succeeded. replicas: {status.replicas}, available: {status.available_replicas }" + ) + return "success" + + if status.replicas > 0 and status.available_replicas is not None and status.available_replicas >= 0: + logger.info( + f"StatefulSet {name} is running. replicas: {status.replicas}, available: {status.available_replicas }" + ) + return "running" + + logger.info( + f"StatefulSet {name} status is unknown. Replicas: {status.replicas}, available: {status.available_replicas }" + ) + return "failed" + except ApiException as e: + logger.error(f"Exception when calling AppsV1Api->read_namespaced_stateful_set: {e}") + return f"Error checking status of StatefulSet {name}: {e}" + + def delete_stateful_set(self, name: str): + try: + self.apps_v1_api.delete_namespaced_stateful_set( + name=name, namespace=self.namespace, body=client.V1DeleteOptions() + ) + logger.info(f"Deleted StatefulSet: {name}") + except ApiException as e: + logger.error(f"Exception when calling AppsV1Api->delete_namespaced_stateful_set: {e}") + + def delete_service(self, name: str): + try: + self.core_v1_api.delete_namespaced_service( + name=name, namespace=self.namespace, body=client.V1DeleteOptions() + ) + logger.info(f"Deleted Service: {name}") + except ApiException as e: + logger.error(f"Exception when calling CoreV1Api->delete_namespaced_service: {e}") + + def get_resources(self) -> client.V1ResourceRequirements: + res = DEFAULT_RESOURCES + flyteidl_limits = self.data_service_config.get("Limits", None) + flyteidl_requests = self.data_service_config.get("Requests", None) + logger.info(f"Flyte Resources: limits: {flyteidl_limits} and requests {flyteidl_requests}") + if flyteidl_limits is not None: + res.limits = convert_flyte_to_k8s_fields(flyteidl_limits) + logger.info(f"Resources limits updated is: {res.limits}") + if flyteidl_requests is not None: + res.requests = convert_flyte_to_k8s_fields(flyteidl_requests) + logger.info(f"Resources requests updated is: {res.requests}") + cleanup_resources(res) + logger.info(f"Resources cleaned up is: {res}") + return res diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/sensor.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/sensor.py new file mode 100644 index 0000000000..ce717b2a0f --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/sensor.py @@ -0,0 +1,67 @@ +from flytekitplugins.k8sdataservice.k8s.kube_config import KubeConfig +from kubernetes import client +from kubernetes.client.rest import ApiException + +from flytekit import logger +from flytekit.sensor.base_sensor import BaseSensor + +TRAININGJOB_API_GROUP = "kubeflow.org" +VERSION = "v1" + + +class CleanupSensor(BaseSensor): + def __init__(self, name: str, namespace: str = "flyte", **kwargs): + """ + Initialize the CleanupSensor class with relevant configurations for monitoring and managing the k8s data service. + """ + super().__init__(name=name, task_type="sensor", **kwargs) + self.k8s_config = KubeConfig() + try: + self.k8s_config.load_kube_config() + except kubernetes.config.ConfigException as e: + logger.error(f"Failed to load kubernetes config: {e}") + raise + self.apps_v1_api = client.AppsV1Api() + self.core_v1_api = client.CoreV1Api() + self.custom_api = client.CustomObjectsApi() + self.namespace = namespace + + async def poke(self, release_name: str, cleanup_data_service: bool, cluster: str) -> bool: + """poke will delete the graph engine resources based on the user's configuration + 1. This has to be done in the control plane by design. We don't expect any users's running pod to be authn/z to manage resources + 2. This can not be done in the async agent because the delete callback is only invoked on abortion operation or failure phase. + while this makes sense but what we need is a separate task to delete graph engine without complicating the regular async agent flow. + 3. In the near future, we will add the poking logic on the training job's status. In the initial implementation, we skipped + it for simplicity. This is also why we use the sensor API to keep forward compatibility + """ + self.release_name = release_name + self.cleanup_data_service = cleanup_data_service + self.cluster = cluster + return await self._handle_cleanup() + + async def _handle_cleanup(self) -> bool: + if not self.cleanup_data_service: + logger.info( + f"User decides to not to clean up the graph engine: {self.release_name} in cluster {self.cluster}, namespace {self.namespace}" + ) + logger.info("DataService sensor will stop polling") + return True + logger.info(f"The training job is in terminal stage, deleting graph engine {self.release_name}") + self.delete_data_service() + return True + + def delete_data_service(self): + """ + Delete the data service's associated Kubernetes resources (StatefulSet and Service). + """ + + def delete_resource(resource_type: str, delete_fn): + try: + delete_fn(name=self.release_name, namespace=self.namespace, body=client.V1DeleteOptions()) + logger.info(f"Deleted {resource_type}: {self.release_name}") + except ApiException as e: + logger.error(f"Error deleting {resource_type}: {e}") + + logger.info(f"Sensor got the release name: {self.release_name}") + delete_resource("Service", self.core_v1_api.delete_namespaced_service) + delete_resource("StatefulSet", self.apps_v1_api.delete_namespaced_stateful_set) diff --git a/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/task.py b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/task.py new file mode 100644 index 0000000000..7888bf7b06 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/task.py @@ -0,0 +1,71 @@ +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Type + +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Struct + +from flytekit import Resources, kwtypes, logger +from flytekit.configuration import SerializationSettings +from flytekit.core.base_task import PythonTask +from flytekit.core.interface import Interface +from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin + + +@dataclass +class DataServiceConfig(object): + """DataServiceConfig should be used to configure a DataServiceTask.""" + + Name: Optional[str] = None + Requests: Optional[Resources] = None + Limits: Optional[Resources] = None + Port: Optional[int] = None + Image: Optional[str] = None + Command: Optional[List[str]] = None + Replicas: Optional[int] = None + ExistingReleaseName: Optional[str] = None + Cluster: Optional[str] = None + + +class DataServiceTask(AsyncAgentExecutorMixin, PythonTask[DataServiceConfig]): + _TASK_TYPE = "dataservicetask" + + def __init__( + self, + name: str, + task_config: Optional[DataServiceConfig], + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + super().__init__( + name=name, + task_config=task_config, + interface=Interface(inputs=inputs, outputs=kwtypes(name=str)), + task_type=self._TASK_TYPE, + **kwargs, + ) + + def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: + logger.info("get_custom is invoked") + config = {} + limits = None + requests = None + if self.task_config is not None: + limits = asdict(self.task_config.Limits) if self.task_config.Limits is not None else None + requests = asdict(self.task_config.Requests) if self.task_config.Requests is not None else None + ge = { + "Name": self.task_config.Name, + "Image": self.task_config.Image, + "Command": self.task_config.Command, + "Port": self.task_config.Port, + "Replicas": self.task_config.Replicas, + "ExistingReleaseName": self.task_config.ExistingReleaseName, + "Cluster": self.task_config.Cluster, + } + if limits is not None: + ge["Limits"] = limits + if requests is not None: + ge["Requests"] = requests + config = ge + s = Struct() + s.update(config) + return json_format.MessageToDict(s) diff --git a/plugins/flytekit-k8sdataservice/setup.py b/plugins/flytekit-k8sdataservice/setup.py new file mode 100644 index 0000000000..e23c0b252b --- /dev/null +++ b/plugins/flytekit-k8sdataservice/setup.py @@ -0,0 +1,38 @@ +from setuptools import find_namespace_packages, setup + +PLUGIN_NAME = "k8sdataservice" + +microlib_name = f"flytekitplugins-{PLUGIN_NAME}" + +plugin_requires = ["flytekit>=1.11.0,<2.0.0", "kubernetes>=23.6.0,<24.0.0", "flyteidl>=1.11.0,<2.0.0"] + +__version__ = "0.0.0+develop" + +setup( + name=microlib_name, + version=__version__, + author="LinkedIn", + author_email="shuliang@linkedin.com", + description="Flytekit K8s Data Service Plugin", + namespace_packages=["flytekitplugins"], + packages=find_namespace_packages(where="."), + include_package_data=True, + install_requires=plugin_requires, + license="apache2", + python_requires=">=3.9", + classifiers=[ + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, +) diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_kube_config.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_kube_config.py new file mode 100644 index 0000000000..bec217d318 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_kube_config.py @@ -0,0 +1,23 @@ +import unittest +from unittest.mock import patch +from flytekitplugins.k8sdataservice.k8s.kube_config import KubeConfig +from kubernetes.config import ConfigException + + +class TestKubeConfig(unittest.TestCase): + + @patch("flytekitplugins.k8sdataservice.k8s.kube_config.config.load_incluster_config") + def test_load_kube_config_success(self, mock_load_incluster_config): + kube_config = KubeConfig() + kube_config.load_kube_config() + mock_load_incluster_config.assert_called_once() + + @patch("flytekitplugins.k8sdataservice.k8s.kube_config.config.load_incluster_config") + def test_load_kube_config_failure(self, mock_load_incluster_config): + # Simulate a ConfigException + mock_load_incluster_config.side_effect = ConfigException("In-cluster config not found.") + kube_config = KubeConfig() + + with self.assertLogs('flytekit', level='WARNING') as log: + kube_config.load_kube_config() + self.assertEqual(f"WARNING:flytekit:Failed to load in-cluster configuration. In-cluster config not found.", log.output[-1]) diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_manager.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_manager.py new file mode 100644 index 0000000000..eebe19bf3f --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/k8s/test_manager.py @@ -0,0 +1,168 @@ +import unittest +from unittest.mock import MagicMock, patch +from flytekitplugins.k8sdataservice.k8s.manager import K8sManager, DEFAULT_RESOURCES +from kubernetes.client import V1ResourceRequirements +from kubernetes.client.rest import ApiException +from kubernetes.client import V1DeleteOptions +from flytekit import logger + + +class TestK8sManager(unittest.TestCase): + + def setUp(self): + self.k8s_manager = K8sManager() + self.k8s_manager.set_configs({ + "Cluster": "ei-dev2", + "Name": "test-name", + "Image": "test-image", + "Command": ["echo", "hello"], + "Replicas": 1, + "Limits": {"cpu": "2", "memory": "4G"}, + "Requests": {"cpu": "1", "memory": "2G"}, + }) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.K8sManager.create_service", return_value="test-service") + @patch("flytekitplugins.k8sdataservice.k8s.manager.K8sManager.create_stateful_set", return_value="test-statefulset") + def test_create_data_service(self, mock_create_stateful_set, mock_create_service): + response = self.k8s_manager.create_data_service() + mock_create_service.assert_called_once() + mock_create_stateful_set.assert_called_once() + self.assertEqual(response, "test-statefulset") + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.create_namespaced_stateful_set") + def test_create_stateful_set(self, mock_create_namespaced_stateful_set): + mock_metadata = MagicMock() + mock_metadata.name = "test-statefulset" + mock_create_namespaced_stateful_set.return_value = MagicMock(metadata=mock_metadata) + stateful_set_object = self.k8s_manager.create_stateful_set_object() + response = self.k8s_manager.create_stateful_set(stateful_set_object) + self.assertEqual(response, "test-statefulset") + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.create_namespaced_stateful_set") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_create_stateful_set_failure(self, mock_logger, mock_create_namespaced_stateful_set): + mock_create_namespaced_stateful_set.side_effect = ApiException("Create failed") + stateful_set_object = self.k8s_manager.create_stateful_set_object() + with self.assertRaises(ApiException): + self.k8s_manager.create_stateful_set(stateful_set_object) + mock_logger.error.assert_called_once() + logged_message = mock_logger.error.call_args[0][0] + self.assertIn("Exception when calling AppsV1Api->create_namespaced_stateful_set: (Create failed)", logged_message) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.CoreV1Api.create_namespaced_service") + def test_create_service(self, mock_create_namespaced_service): + # Set the return value of the mock to simulate the actual response + mock_metadata = MagicMock() + mock_metadata.name = "test-service" + mock_create_namespaced_service.return_value = MagicMock(metadata=mock_metadata) + # Call the method and assert the result + response = self.k8s_manager.create_service() + self.assertEqual(response, "test-service") + + # Test with empty metadata + mock_create_namespaced_service.return_value = MagicMock(metadata=None) + with self.assertRaises(ValueError): + self.k8s_manager.create_service() + + # Test with None service name + mock_metadata = MagicMock() + mock_metadata.name = None + mock_create_namespaced_service.return_value = MagicMock(metadata=mock_metadata) + self.assertEqual(self.k8s_manager.create_service(), None) + + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.CoreV1Api.create_namespaced_service") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_create_service_exception(self, mock_logger, mock_create_namespaced_service): + mock_create_namespaced_service.side_effect = ApiException("Failed to create service") + with self.assertRaises(ApiException): + self.k8s_manager.create_service() + mock_logger.error.assert_called_once() + logged_message = mock_logger.error.call_args[0][0] + self.assertIn("Exception when calling CoreV1Api->create_namespaced_service: (Failed to create service)", logged_message) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.read_namespaced_stateful_set") + def test_check_stateful_set_status(self, mock_read_namespaced_stateful_set): + mock_read_namespaced_stateful_set.return_value.status = MagicMock(replicas=1, available_replicas=1) + self.assertEqual(self.k8s_manager.check_stateful_set_status("test-name"), "success") + + mock_read_namespaced_stateful_set.return_value.status = MagicMock(replicas=1, available_replicas=0) + self.assertEqual(self.k8s_manager.check_stateful_set_status("test-name"), "running") + + mock_read_namespaced_stateful_set.return_value.status = MagicMock(replicas=0, available_replicas=0) + self.assertEqual(self.k8s_manager.check_stateful_set_status("test-name"), "pending") + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.delete_namespaced_stateful_set") + def test_delete_stateful_set(self, mock_delete_stateful_set): + self.k8s_manager.namespace = "kk-flyte-dev2" + self.k8s_manager.delete_stateful_set("test-name") + mock_delete_stateful_set.assert_called_once_with(name="test-name", namespace="kk-flyte-dev2", body=V1DeleteOptions()) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.CoreV1Api.delete_namespaced_service") + def test_delete_service(self, mock_delete_service): + self.k8s_manager.namespace = "kk-flyte-dev2" + self.k8s_manager.delete_service("test-name") + mock_delete_service.assert_called_once_with(name="test-name", namespace="kk-flyte-dev2", body=V1DeleteOptions()) + + def test_get_resources(self): + resources = self.k8s_manager.get_resources() + assert self.k8s_manager.data_service_config.get("Limits") is not None + self.assertIsInstance(resources, V1ResourceRequirements) + self.assertEqual(resources.limits, {"cpu": "2", "memory": "4G"}) + self.assertEqual(resources.requests, {"cpu": "1", "memory": "2G"}) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.read_namespaced_stateful_set") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_check_stateful_set_status_exception(self, mock_logger, mock_read_namespaced_stateful_set): + mock_read_namespaced_stateful_set.side_effect = ApiException("Failed to read StatefulSet") + result = self.k8s_manager.check_stateful_set_status("test-statefulset") + expected_message = "Error checking status of StatefulSet test-statefulset: (Failed to read StatefulSet)" + self.assertIn(expected_message, result) + mock_logger.error.assert_called_once() + logged_message = mock_logger.error.call_args[0][0] + self.assertIn("Exception when calling AppsV1Api->read_namespaced_stateful_set: (Failed to read StatefulSet)", logged_message) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.read_namespaced_stateful_set") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_check_stateful_set_status_unknown(self, mock_logger, mock_read_namespaced_stateful_set): + mock_status = MagicMock() + mock_status.replicas = 3 + mock_status.available_replicas = None + mock_read_namespaced_stateful_set.return_value.status = mock_status + result = self.k8s_manager.check_stateful_set_status("test-statefulset") + self.assertEqual(result, "failed") + mock_logger.info.assert_any_call("StatefulSet test-statefulset status is unknown. Replicas: 3, available: None") + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.CoreV1Api.delete_namespaced_service") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_delete_service_exception(self, mock_logger, mock_delete_service): + mock_delete_service.side_effect = ApiException("Failed to delete Service") + self.k8s_manager.delete_service("test-service") + mock_logger.error.assert_called_once() + logged_message = mock_logger.error.call_args[0][0] + self.assertIn("Exception when calling CoreV1Api->delete_namespaced_service: (Failed to delete Service)", logged_message) + + @patch("flytekitplugins.k8sdataservice.k8s.manager.client.AppsV1Api.delete_namespaced_stateful_set") + @patch("flytekitplugins.k8sdataservice.k8s.manager.logger") + def test_delete_stateful_set_exception(self, mock_logger, mock_delete_stateful_set): + mock_delete_stateful_set.side_effect = ApiException("Failed to delete StatefulSet") + self.k8s_manager.delete_stateful_set("test-statefulset") + mock_logger.error.assert_called_once() + logged_message = mock_logger.error.call_args[0][0] + self.assertIn("Exception when calling AppsV1Api->delete_namespaced_stateful_set: (Failed to delete StatefulSet)", logged_message) + + def test_get_resources_default(self): + self.k8s_manager.set_configs({ + "Cluster": "ei-dev2", + "Name": "test-name", + "Image": "test-image", + "Command": ["echo", "hello"], + "Replicas": 1, + "Limits": {"cpu": "6", "memory": "16G"}, + "Requests": {"cpu": "2", "memory": "10G"}, + }) + resources = self.k8s_manager.get_resources() + logger.info(f"After Testing default resources: {DEFAULT_RESOURCES}") + self.assertIsInstance(resources, V1ResourceRequirements) + self.assertEqual(resources.limits, {"cpu": "6", "memory": "16G"}) + self.assertEqual(resources.requests, {"cpu": "2", "memory": "10G"}) diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_agent.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_agent.py new file mode 100644 index 0000000000..0db9877c26 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_agent.py @@ -0,0 +1,319 @@ +import json +from dataclasses import asdict +from datetime import timedelta +from unittest.mock import patch, MagicMock +import grpc +from google.protobuf import json_format +from flytekitplugins.k8sdataservice.task import DataServiceConfig +from flytekitplugins.k8sdataservice.agent import DataServiceMetadata +from google.protobuf.struct_pb2 import Struct +from flyteidl.core.execution_pb2 import TaskExecution +import flytekit.models.interface as interface_models +from flytekit.extend.backend.base_agent import AgentRegistry +from flytekit.interfaces.cli_identifiers import Identifier +from flytekit.models import literals, task, types +from flytekit.models.core.identifier import ResourceType +from flytekit.models.task import TaskTemplate + + +cmd = ["command", "args"] + + +def create_test_task_metadata() -> task.TaskMetadata: + return task.TaskMetadata( + discoverable= True, + runtime=task.RuntimeMetadata(task.RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"), + timeout=timedelta(days=1), + retries=literals.RetryStrategy(3), + interruptible=True, + discovery_version="0.1.1b0", + deprecated_error_message="This is deprecated!", + cache_serializable=True, + pod_template_name="A", + cache_ignore_input_vars=(), + ) + + +def create_test_setup(original_name: str = "gnn-1234", existing_release_name: str = "gnn-2345"): + task_id = Identifier( + resource_type=ResourceType.TASK, project="project", domain="domain", name="name", version="version" + ) + task_metadata = create_test_task_metadata() + s = Struct() + if existing_release_name != "": + s.update({ + "Name": original_name, + "Image": "image", + "Command": cmd, + "Cluster": "ei-dev2", + "ExistingReleaseName": existing_release_name, + }) + else: + s.update({ + "Name": original_name, + "Image": "image", + "Command": cmd, + "Cluster": "ei-dev2", + }) + task_config = json_format.MessageToDict(s) + return task_id, task_metadata, task_config + + +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.create_data_service", return_value="gnn-1234") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.check_stateful_set_status", return_value="succeeded") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_stateful_set") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_service") +def test_gnn_agent(mock_delete_service, mock_delete_stateful_set, mock_check_status, mock_create_data_service): + agent = AgentRegistry.get_agent("dataservicetask") + task_id, task_metadata, task_config = create_test_setup(existing_release_name="") + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + { + "a": interface_models.Variable(int_type, "description1"), + "b": interface_models.Variable(int_type, "description2"), + }, + {}, + ) + task_inputs = literals.LiteralMap( + { + "a": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + "b": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + }, + ) + + dummy_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=interfaces, + type="dataservicetask", + ) + + expected_resource_metadata = DataServiceMetadata( + dataservice_config=DataServiceConfig(Name="gnn-1234", Image="image", Command=cmd, Cluster="ei-dev2"), + name="gnn-1234") + # Test create method + res_resource_metadata = agent.create(dummy_template, task_inputs) + assert res_resource_metadata == expected_resource_metadata + mock_create_data_service.assert_called_once() + + # Test get method + res = agent.get(res_resource_metadata) + assert res.phase == TaskExecution.SUCCEEDED + assert res.outputs.get("data_service_name") == 'gnn-1234' + mock_check_status.assert_called_once_with("gnn-1234") + + # # Test delete method + agent.delete(res_resource_metadata) + mock_delete_stateful_set.assert_called_once_with("gnn-1234") + mock_delete_service.assert_called_once_with("gnn-1234") + + +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.create_data_service", return_value="gnn-1234") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.check_stateful_set_status", return_value="succeeded") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_stateful_set") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_service") +def test_gnn_agent_reuse_data_service(mock_delete_service, mock_delete_stateful_set, mock_check_status, mock_create_data_service): + agent = AgentRegistry.get_agent("dataservicetask") + task_id, task_metadata, task_config = create_test_setup(original_name="gnn-2345", existing_release_name="gnn-2345") + + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + { + "a": interface_models.Variable(int_type, "description1"), + "b": interface_models.Variable(int_type, "description2"), + }, + {}, + ) + task_inputs = literals.LiteralMap( + { + "a": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + "b": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + }, + ) + + dummy_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=interfaces, + type="dataservicetask", + ) + + expected_resource_metadata = DataServiceMetadata( + dataservice_config=DataServiceConfig( + Name="gnn-2345", Image="image", Command=cmd, Cluster="ei-dev2", ExistingReleaseName="gnn-2345"), + name="gnn-2345") + + # Test create method, and create_data_service should have not been called + res_resource_metadata = agent.create(dummy_template, task_inputs) + assert res_resource_metadata == expected_resource_metadata + mock_create_data_service.assert_not_called() + + # Test get method + res = agent.get(res_resource_metadata) + assert res.phase == TaskExecution.SUCCEEDED + mock_check_status.assert_called_once_with("gnn-2345") + + # # Test delete method + agent.delete(res_resource_metadata) + mock_delete_stateful_set.assert_called_once_with("gnn-2345") + mock_delete_service.assert_called_once_with("gnn-2345") + + +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.create_data_service", return_value="gnn-1234") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.check_stateful_set_status", return_value="running") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_stateful_set") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_service") +def test_gnn_agent_status(mock_delete_service, mock_delete_stateful_set, mock_check_status, mock_create_data_service): + agent = AgentRegistry.get_agent("dataservicetask") + task_id, task_metadata, task_config = create_test_setup(original_name="gnn-2345", existing_release_name="gnn-2345") + + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + { + "a": interface_models.Variable(int_type, "description1"), + "b": interface_models.Variable(int_type, "description2"), + }, + {}, + ) + task_inputs = literals.LiteralMap( + { + "a": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + "b": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + }, + ) + + dummy_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=interfaces, + type="dataservicetask", + ) + + expected_resource_metadata = DataServiceMetadata( + dataservice_config=DataServiceConfig( + Name="gnn-2345", Image="image", Command=cmd, Cluster="ei-dev2", ExistingReleaseName="gnn-2345"), + name="gnn-2345") + # Test create method, and create_data_service should have not been called + res_resource_metadata = agent.create(dummy_template, task_inputs) + assert res_resource_metadata == expected_resource_metadata + mock_create_data_service.assert_not_called() + + # Test get method + res = agent.get(res_resource_metadata) + assert res.phase == TaskExecution.RUNNING + mock_check_status.assert_called_once_with("gnn-2345") + + # # Test delete methods are not called + agent.delete(res_resource_metadata) + mock_delete_stateful_set.assert_called_once_with("gnn-2345") + mock_delete_service.assert_called_once_with("gnn-2345") + + +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.create_data_service", return_value="gnn-1234") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.check_stateful_set_status", return_value="succeeded") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_stateful_set") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_service") +def test_gnn_agent_no_configmap(mock_delete_service, mock_delete_stateful_set, mock_check_status, mock_create_data_service): + agent = AgentRegistry.get_agent("dataservicetask") + task_id, task_metadata, task_config = create_test_setup(original_name="gnn-2345", existing_release_name="gnn-2345") + + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + { + "a": interface_models.Variable(int_type, "description1"), + "b": interface_models.Variable(int_type, "description2"), + }, + {}, + ) + task_inputs = literals.LiteralMap( + { + "a": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + "b": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + }, + ) + + dummy_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=interfaces, + type="dataservicetask", + ) + + expected_resource_metadata = DataServiceMetadata( + dataservice_config=DataServiceConfig( + Name="gnn-2345", Image="image", Command=cmd, Cluster="ei-dev2", ExistingReleaseName="gnn-2345"), + name="gnn-2345") + + # Test create method, and create_data_service should have not been called + res_resource_metadata = agent.create(dummy_template, task_inputs) + assert res_resource_metadata == expected_resource_metadata + mock_create_data_service.assert_not_called() + + # Test get method + res = agent.get(res_resource_metadata) + assert res.phase == TaskExecution.SUCCEEDED + mock_check_status.assert_called_once_with("gnn-2345") + + # # Test delete methods are not called + agent.delete(res_resource_metadata) + mock_delete_stateful_set.assert_called_once_with("gnn-2345") + mock_delete_service.assert_called_once_with("gnn-2345") + + +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.create_data_service", return_value="gnn-1234") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.check_stateful_set_status", return_value="pending") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_stateful_set") +@patch("flytekitplugins.k8sdataservice.agent.K8sManager.delete_service") +def test_gnn_agent_status_failed(mock_delete_service, mock_delete_stateful_set, mock_check_status, mock_create_data_service): + agent = AgentRegistry.get_agent("dataservicetask") + task_id, task_metadata, task_config = create_test_setup(original_name="gnn-2345", existing_release_name="gnn-2345") + + int_type = types.LiteralType(types.SimpleType.INTEGER) + interfaces = interface_models.TypedInterface( + { + "a": interface_models.Variable(int_type, "description1"), + "b": interface_models.Variable(int_type, "description2"), + }, + {}, + ) + task_inputs = literals.LiteralMap( + { + "a": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + "b": literals.Literal(scalar=literals.Scalar(primitive=literals.Primitive(integer=1))), + }, + ) + + dummy_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=interfaces, + type="dataservicetask", + ) + + expected_resource_metadata = DataServiceMetadata( + dataservice_config=DataServiceConfig( + Name="gnn-2345", Image="image", Command=cmd, Cluster="ei-dev2", ExistingReleaseName="gnn-2345"), + name="gnn-2345") + + # Test create method, and create_data_service should have not been called + res_resource_metadata = agent.create(dummy_template, task_inputs) + assert res_resource_metadata == expected_resource_metadata + mock_create_data_service.assert_not_called() + + # Test get method + res = agent.get(res_resource_metadata) + assert res.phase == TaskExecution.RUNNING + mock_check_status.assert_called_once_with("gnn-2345") + + mock_check_status.return_value = "failed" + res.phase == TaskExecution.FAILED + + # # Test delete methods are not called + agent.delete(res_resource_metadata) + mock_delete_stateful_set.assert_called_once_with("gnn-2345") + mock_delete_service.assert_called_once_with("gnn-2345") diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_sensor.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_sensor.py new file mode 100644 index 0000000000..3594e04ac9 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_sensor.py @@ -0,0 +1,134 @@ +import pytest +import asyncio +import unittest +from unittest.mock import patch, MagicMock +from kubernetes import client +from kubernetes.client.rest import ApiException +from flytekitplugins.k8sdataservice.sensor import CleanupSensor + + +@pytest.mark.asyncio +@patch("flytekitplugins.k8sdataservice.sensor.KubeConfig") +@patch("flytekitplugins.k8sdataservice.sensor.client.AppsV1Api") +@patch("flytekitplugins.k8sdataservice.sensor.client.CoreV1Api") +@patch("flytekitplugins.k8sdataservice.sensor.client.CustomObjectsApi") +@patch("flytekitplugins.k8sdataservice.sensor.logger") +async def test_poke_no_cleanup( + mock_logger, + mock_custom_api, + mock_core_v1_api, + mock_apps_v1_api, + mock_kube_config +): + mock_kube_config.return_value = MagicMock() + sensor = CleanupSensor(name="test-sensor") + await asyncio.create_task( + sensor.poke(release_name="test-release", + cleanup_data_service=False, + cluster="test-cluster") + ) + mock_kube_config.return_value.load_kube_config.assert_called_once() + assert isinstance(sensor.apps_v1_api, MagicMock) + assert isinstance(sensor.core_v1_api, MagicMock) + assert isinstance(sensor.custom_api, MagicMock) + assert sensor.release_name == "test-release" + assert sensor.cleanup_data_service is False + assert sensor.namespace == "flyte" + assert sensor.cluster == "test-cluster" + + +@pytest.mark.asyncio +@patch("flytekitplugins.k8sdataservice.sensor.CleanupSensor.delete_data_service") +@patch("flytekitplugins.k8sdataservice.sensor.KubeConfig") +@patch("flytekitplugins.k8sdataservice.sensor.client.AppsV1Api") +@patch("flytekitplugins.k8sdataservice.sensor.client.CoreV1Api") +@patch("flytekitplugins.k8sdataservice.sensor.client.CustomObjectsApi") +@patch("flytekitplugins.k8sdataservice.sensor.logger") +async def test_poke_with_cleanup( + mock_logger, + mock_custom_api, + mock_core_v1_api, + mock_apps_v1_api, + mock_kube_config, + mock_delete_data_service +): + mock_kube_config.return_value = MagicMock() + # Initialize CleanupSensor instance + sensor = CleanupSensor(name="test-sensor") + + async def sensor_poke_task(): + await sensor.poke( + release_name="test-release", + cleanup_data_service=True, + cluster="test-cluster" + ) + + # Use asyncio.wait_for for timeout handling (compatible with older Python versions) + try: + await asyncio.wait_for(sensor_poke_task(), timeout=30) + except asyncio.TimeoutError: + pytest.fail("Test timed out") + + # Assertions for logger and delete_data_service call + mock_logger.info.assert_any_call( + "The training job is in terminal stage, deleting graph engine test-release" + ) + mock_delete_data_service.assert_called_once() + + +class TestCleanupSensor(unittest.TestCase): + @patch("flytekitplugins.k8sdataservice.sensor.logger") + @patch("flytekitplugins.k8sdataservice.sensor.client.CoreV1Api") + @patch("flytekitplugins.k8sdataservice.sensor.client.AppsV1Api") + def test_delete_data_service(self, mock_apps_v1_api, mock_core_v1_api, mock_logger): + # Configure mocks for service deletion + mock_core_v1_api_instance = mock_core_v1_api.return_value + mock_apps_v1_api_instance = mock_apps_v1_api.return_value + # Initialize sensor and set required properties + sensor = CleanupSensor(name="test-sensor") + sensor.core_v1_api = mock_core_v1_api_instance + sensor.apps_v1_api = mock_apps_v1_api_instance + sensor.release_name = "test-release" + sensor.namespace = "test-namespace" + + # Call delete_data_service + sensor.delete_data_service() + + # Verify the service deletion calls + mock_core_v1_api_instance.delete_namespaced_service.assert_called_once_with( + name="test-release", namespace="test-namespace", body=client.V1DeleteOptions() + ) + mock_apps_v1_api_instance.delete_namespaced_stateful_set.assert_called_once_with( + name="test-release", namespace="test-namespace", body=client.V1DeleteOptions() + ) + # Check for logger messages indicating success + mock_logger.info.assert_any_call("Deleted Service: test-release") + mock_logger.info.assert_any_call("Deleted StatefulSet: test-release") + + @patch("flytekitplugins.k8sdataservice.sensor.logger") + @patch("flytekitplugins.k8sdataservice.sensor.client.CoreV1Api") + @patch("flytekitplugins.k8sdataservice.sensor.client.AppsV1Api") + def test_delete_data_service_with_exceptions(self, mock_apps_v1_api, mock_core_v1_api, mock_logger): + # Configure mocks to raise exceptions + mock_core_v1_api_instance = mock_core_v1_api.return_value + mock_apps_v1_api_instance = mock_apps_v1_api.return_value + + mock_core_v1_api_instance.delete_namespaced_service.side_effect = ApiException("Service deletion failed") + mock_apps_v1_api_instance.delete_namespaced_stateful_set.side_effect = ApiException("StatefulSet deletion failed") + + # Initialize sensor and set required properties + sensor = CleanupSensor(name="test-sensor") + sensor.core_v1_api = mock_core_v1_api_instance + sensor.apps_v1_api = mock_apps_v1_api_instance + sensor.release_name = "test-release" + sensor.namespace = "test-namespace" + + # Call delete_data_service and handle exceptions + sensor.delete_data_service() + + # Verify that errors were logged + service_error_logged = any("Error deleting Service" in call[0][0] for call in mock_logger.error.call_args_list) + statefulset_error_logged = any("Error deleting StatefulSet" in call[0][0] for call in mock_logger.error.call_args_list) + + self.assertTrue(service_error_logged, "Service deletion error not logged as expected") + self.assertTrue(statefulset_error_logged, "StatefulSet deletion error not logged as expected") diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_task.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_task.py new file mode 100644 index 0000000000..0c8e4cc85a --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/test_task.py @@ -0,0 +1,128 @@ +from collections import OrderedDict +from unittest.mock import patch, MagicMock, mock_open +import pytest +from flytekitplugins.k8sdataservice.task import DataServiceConfig, DataServiceTask +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Struct +from flytekit import kwtypes, Resources +from flytekit.configuration import ImageConfig, SerializationSettings +from flytekit.extend import get_serializable + + +@pytest.fixture +def mock_k8s_manager(): + with patch('flytekitplugins.k8sdataservice.agent.K8sManager') as MockK8sManager: + mock_k8s_manager = MagicMock() + MockK8sManager.return_value = mock_k8s_manager + yield mock_k8s_manager + + +def test_gnn_task(): + gnn_config = DataServiceConfig( + Name="test", + Replicas=8, + Image="image", + Command=None, + Cluster="grid2", + Requests=Resources(cpu="1", mem="2", gpu="4"), + Limits=Resources(cpu="1", mem="2", gpu="4"), + Port=1234, + ) + gnn_task = DataServiceTask( + name="flytekit.poc.gnn.task", + inputs=kwtypes(ds=str), + task_config=gnn_config, + ) + + serialization_settings = SerializationSettings( + project="test", + domain="lnkd", + image_config=ImageConfig.auto(), + env={}, + ) + + task_spec = get_serializable(OrderedDict(), serialization_settings, gnn_task) + s = Struct() + s.update({ + "Name": "test", + "Replicas": 8, + "Image": "image", + "Command": None, + "Cluster": "grid2", + "ExistingReleaseName": None, + "Requests": { + "cpu": "1", + "mem": "2", + "gpu": "4", + "ephemeral_storage": None + }, + "Limits": { + "cpu": "1", + "mem": "2", + "gpu": "4", + "ephemeral_storage": None + }, + "Port": 1234, + }) + assert task_spec.template.custom == json_format.MessageToDict(s) + + +def test_gnn_task_optional_field(): + gnn_config = DataServiceConfig( + Name="test", + Replicas=8, + Image="image", + Command=None, + Cluster="grid2", + Port=1234, + ) + gnn_task = DataServiceTask( + name="flytekit.poc.gnn.task", + inputs=kwtypes(ds=str), + task_config=gnn_config, + ) + + serialization_settings = SerializationSettings( + project="test", + domain="lnkd", + image_config=ImageConfig.auto(), + env={}, + ) + + task_spec = get_serializable(OrderedDict(), serialization_settings, gnn_task) + s = Struct() + s.update({ + "Name": "test", + "Replicas": 8, + "Image": "image", + "Command": None, + "Cluster": "grid2", + "ExistingReleaseName": None, + "Port": 1234, + }) + assert task_spec.template.custom == json_format.MessageToDict(s) + + +def test_local_exec(mock_k8s_manager): + gnn_config = DataServiceConfig( + Replicas=8, + Image="image", + Command=[ + "bash", + "-c", + "command", + ], + Cluster="grid2", + ) + gnn_task = DataServiceTask( + name="flytekit.poc.gnn.task", + inputs=kwtypes(ds=str), + task_config=gnn_config, + ) + assert gnn_task is not None + assert len(gnn_task.interface.inputs) == 1 + assert len(gnn_task.interface.outputs) == 1 + + # will not run locally + with pytest.raises(Exception): + gnn_task(ds="GNNTEST") diff --git a/plugins/flytekit-k8sdataservice/tests/k8sdataservice/utils/test_resources.py b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/utils/test_resources.py new file mode 100644 index 0000000000..ac04c610d6 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/tests/k8sdataservice/utils/test_resources.py @@ -0,0 +1,90 @@ +import unittest +from kubernetes import client +from utils.resources import cleanup_resources, convert_flyte_to_k8s_fields + + +class TestCleanupResources(unittest.TestCase): + + def test_cleanup_resources(self): + resources = client.V1ResourceRequirements( + limits={"cpu": "2", "memory": "10G", "ephemeral_storage": "0", "storage": "0GB"}, + requests={"cpu": "1", "memory": "0", "ephemeral_storage": "5G", "storage": "0"} + ) + cleanup_resources(resources) + expected_limits = {"cpu": "2", "memory": "10G"} + self.assertEqual(resources.limits, expected_limits) + + expected_requests = {"cpu": "1", "ephemeral_storage": "5G"} + self.assertEqual(resources.requests, expected_requests) + + def test_cleanup_resources_no_zero_like_values(self): + resources = client.V1ResourceRequirements( + limits={"cpu": "4", "memory": "16G"}, + requests={"cpu": "2", "memory": "8G"} + ) + + cleanup_resources(resources) + + expected_limits = {"cpu": "4", "memory": "16G"} + expected_requests = {"cpu": "2", "memory": "8G"} + + self.assertEqual(resources.limits, expected_limits) + self.assertEqual(resources.requests, expected_requests) + + def test_cleanup_resources_all_zero_like_values(self): + resources = client.V1ResourceRequirements( + limits={"cpu": "0", "memory": "0GB", "ephemeral_storage": "0"}, + requests={"cpu": "0", "memory": "0", "ephemeral_storage": "0GB"} + ) + + cleanup_resources(resources) + + self.assertEqual(resources.limits, {}) + self.assertEqual(resources.requests, {}) + + def test_cleanup_resources_none_value(self): + resources = client.V1ResourceRequirements( + limits={"cpu": "4", "memory": "16G", "ephemeral_storage": None}, + requests={"cpu": "2", "memory": "8", "storage": None} + ) + + cleanup_resources(resources) + + expected_limits = {"cpu": "4", "memory": "16G"} + expected_requests = {"cpu": "2", "memory": "8"} + + self.assertEqual(resources.limits, expected_limits) + self.assertEqual(resources.requests, expected_requests) + + +class TestConvertFlyteToK8sFields(unittest.TestCase): + + def test_convert_flyte_to_k8s_fields(self): + input_dict = { + "cpu": "2", + "mem": "10G", + "ephemeral_storage": "50G" + } + + expected_output = { + "cpu": "2", + "memory": "10G", + "ephemeral_storage": "50G" + } + + result = convert_flyte_to_k8s_fields(input_dict) + self.assertEqual(result, expected_output) + + def test_no_mem_key(self): + input_dict = { + "cpu": "1", + "storage": "100G" + } + + expected_output = { + "cpu": "1", + "storage": "100G" + } + + result = convert_flyte_to_k8s_fields(input_dict) + self.assertEqual(result, expected_output) diff --git a/plugins/flytekit-k8sdataservice/utils/__init__.py b/plugins/flytekit-k8sdataservice/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-k8sdataservice/utils/infra.py b/plugins/flytekit-k8sdataservice/utils/infra.py new file mode 100644 index 0000000000..734cfe0613 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/utils/infra.py @@ -0,0 +1,9 @@ +import hashlib +import uuid + + +def gen_infra_name() -> str: + random_uuid = uuid.uuid4().hex + hash_object = hashlib.sha256(random_uuid.encode()) + hash_value = hash_object.hexdigest()[:20] + return f"flyte-k8sdsinfra-{hash_value}" diff --git a/plugins/flytekit-k8sdataservice/utils/resources.py b/plugins/flytekit-k8sdataservice/utils/resources.py new file mode 100644 index 0000000000..74db27fb48 --- /dev/null +++ b/plugins/flytekit-k8sdataservice/utils/resources.py @@ -0,0 +1,24 @@ +from kubernetes import client + + +def cleanup_resources(resources: client.V1ResourceRequirements): + filtered_resources_limits = {k: v for k, v in resources.limits.items() if not is_zero_like(v)} + filtered_resources_requests = {k: v for k, v in resources.requests.items() if not is_zero_like(v)} + resources.limits = filtered_resources_limits + resources.requests = filtered_resources_requests + + +# We have noticed that for the fields that are not specified, there will be 0 or "0" filled in +# this is helper function to filter those values out. +def is_zero_like(value): + return ( + value is None + or value == "0" + or value == 0 + or str(value).strip().lower() == "0" + or str(value).strip().lower() == "0gb" + ) + + +def convert_flyte_to_k8s_fields(resources_dict): + return {("memory" if k == "mem" else k): v for k, v in resources_dict.items()} diff --git a/plugins/setup.py b/plugins/setup.py index 8f042a9d3a..c04bd7a780 100644 --- a/plugins/setup.py +++ b/plugins/setup.py @@ -29,6 +29,7 @@ "flytekitplugins-huggingface": "flytekit-huggingface", "flytekitplugins-inference": "flytekit-inference", "flytekitplugins-pod": "flytekit-k8s-pod", + "flytekitplugins-k8sdataservice": "flytekit-k8sdataservice", "flytekitplugins-kfmpi": "flytekit-kf-mpi", "flytekitplugins-kfpytorch": "flytekit-kf-pytorch", "flytekitplugins-kftensorflow": "flytekit-kf-tensorflow", diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 82c18b3c50..cef154b6d8 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -124,6 +124,15 @@ def test_pydantic_default_input_with_map_task(): assert execution.closure.phase == WorkflowExecutionPhase.SUCCEEDED, f"Execution failed with phase: {execution.closure.phase}" +def test_pydantic_default_input_with_map_task(): + execution_id = run("pydantic_wf.py", "wf") + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + execution = remote.fetch_execution(name=execution_id) + execution = remote.wait(execution=execution, timeout=datetime.timedelta(minutes=5)) + print("Execution Error:", execution.error) + assert execution.closure.phase == WorkflowExecutionPhase.SUCCEEDED, f"Execution failed with phase: {execution.closure.phase}" + + def test_generic_idl_flytetypes(): os.environ["FLYTE_USE_OLD_DC_FORMAT"] = "true" # default inputs for flyte types in dataclass diff --git a/tests/flytekit/unit/core/test_generice_idl_type_engine.py b/tests/flytekit/unit/core/test_generice_idl_type_engine.py index be3b735e64..0a542627ca 100644 --- a/tests/flytekit/unit/core/test_generice_idl_type_engine.py +++ b/tests/flytekit/unit/core/test_generice_idl_type_engine.py @@ -3008,18 +3008,18 @@ class Config(BaseConfig): include_subtypes=True, ) - subclass_type: SubclassTypes = SubclassTypes.BASE base_attribute: int + subclass_type: SubclassTypes = SubclassTypes.BASE @dataclass(kw_only=True) class ClassA(BaseClass): + class_a_attribute: str # type: ignore[misc] subclass_type: SubclassTypes = SubclassTypes.CLASS_A - class_a_attribute: str @dataclass(kw_only=True) class ClassB(BaseClass): + class_b_attribute: float # type: ignore[misc] subclass_type: SubclassTypes = SubclassTypes.CLASS_B - class_b_attribute: float @task def assert_class_and_return(instance: BaseClass) -> BaseClass: diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 5f5686b3f9..08fd482614 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -3018,18 +3018,18 @@ class Config(BaseConfig): include_subtypes=True, ) - subclass_type: SubclassTypes = SubclassTypes.BASE base_attribute: int + subclass_type: SubclassTypes = SubclassTypes.BASE @dataclass(kw_only=True) class ClassA(BaseClass): + class_a_attribute: str # type: ignore[misc] subclass_type: SubclassTypes = SubclassTypes.CLASS_A - class_a_attribute: str @dataclass(kw_only=True) class ClassB(BaseClass): + class_b_attribute: float # type: ignore[misc] subclass_type: SubclassTypes = SubclassTypes.CLASS_B - class_b_attribute: float @task def assert_class_and_return(instance: BaseClass) -> BaseClass: From 06db981a5b8e687adb360131f27793f4069edd04 Mon Sep 17 00:00:00 2001 From: adrianloy Date: Thu, 23 Jan 2025 02:13:29 +0100 Subject: [PATCH 17/27] Bugfix Omegaconf plugin: Properly deal with NoneType values (#3056) * Treat builtins.NoneType explicitly when parsing type descriptions Signed-off-by: Adrian Loy * Modified tests to cover NoneType case Signed-off-by: Adrian Loy --------- Signed-off-by: Adrian Loy --- .../flytekitplugins/omegaconf/dictconfig_transformer.py | 2 ++ .../tests/test_dictconfig_transformer.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/dictconfig_transformer.py b/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/dictconfig_transformer.py index 0f2b8c63cc..52827f9baa 100644 --- a/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/dictconfig_transformer.py +++ b/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/dictconfig_transformer.py @@ -143,6 +143,8 @@ def parse_type_description(type_desc: str) -> Type: return origin[sub_types[0]] return origin[tuple(sub_types)] else: + if type_desc == "builtins.NoneType": + return NoneType module_name, class_name = type_desc.rsplit(".", 1) return importlib.import_module(module_name).__getattribute__(class_name) diff --git a/plugins/flytekit-omegaconf/tests/test_dictconfig_transformer.py b/plugins/flytekit-omegaconf/tests/test_dictconfig_transformer.py index b4d9115fa9..6ab531cb22 100644 --- a/plugins/flytekit-omegaconf/tests/test_dictconfig_transformer.py +++ b/plugins/flytekit-omegaconf/tests/test_dictconfig_transformer.py @@ -5,7 +5,7 @@ check_if_valid_dictconfig, extract_type_and_value_maps, is_flattenable, - parse_type_description, + parse_type_description, NoneType, ) from omegaconf import DictConfig, OmegaConf @@ -77,11 +77,11 @@ def test_is_flattenable(config: DictConfig, should_flatten: bool, monkeypatch: p def test_extract_type_and_value_maps_simple() -> None: """Test extraction of type and value maps from a simple DictConfig.""" ctx = FlyteContext.current_context() - config: DictConfig = OmegaConf.create({"key1": "value1", "key2": 123, "key3": True}) + config: DictConfig = OmegaConf.create({"key1": "value1", "key2": 123, "key3": True, "key4": None}) type_map, value_map = extract_type_and_value_maps(ctx, config) - expected_type_map = {"key1": "builtins.str", "key2": "builtins.int", "key3": "builtins.bool"} + expected_type_map = {"key1": "builtins.str", "key2": "builtins.int", "key3": "builtins.bool", "key4": "builtins.NoneType"} assert type_map == expected_type_map assert "key1" in value_map @@ -93,6 +93,7 @@ def test_extract_type_and_value_maps_simple() -> None: "type_desc, expected_type", [ ("builtins.int", int), + ("builtins.NoneType", NoneType), ("typing.List[builtins.int]", t.List[int]), ("typing.Optional[builtins.int]", t.Optional[int]), ], From f2a1742eef8f3535e69620d5300f09c1bfaa3174 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 23 Jan 2025 15:00:17 -0500 Subject: [PATCH 18/27] Adds secret env_var (#3048) * Adds secret env name Signed-off-by: Thomas J. Fan * Use env_var Signed-off-by: Thomas J. Fan * Bump flyteidl Signed-off-by: Thomas J. Fan * Smaller change Signed-off-by: Thomas J. Fan * Add integration test Signed-off-by: Thomas J. Fan * Add integration test Signed-off-by: Thomas J. Fan * Use env_var Signed-off-by: Thomas J. Fan * Check for file and env_var Signed-off-by: Thomas J. Fan * Add check for kubectl Signed-off-by: Thomas J. Fan --------- Signed-off-by: Thomas J. Fan --- flytekit/models/security.py | 6 +++ pyproject.toml | 2 +- .../integration/remote/test_remote.py | 47 +++++++++++++++++++ .../remote/workflows/basic/get_secret.py | 26 ++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/flytekit/integration/remote/workflows/basic/get_secret.py diff --git a/flytekit/models/security.py b/flytekit/models/security.py index e210c910b7..be1c586d54 100644 --- a/flytekit/models/security.py +++ b/flytekit/models/security.py @@ -17,6 +17,9 @@ class Secret(_common.FlyteIdlEntity): key is optional and can be an individual secret identifier within the secret For k8s this is required version is the version of the secret. This is an optional field mount_requirement provides a hint to the system as to how the secret should be injected + env_var is optional. Custom environment name to set the value of the secret. + If mount_requirement is ENV_VAR, then the value is the secret itself. + If mount_requirement is FILE, then the value is the path to the secret file. """ class MountType(Enum): @@ -39,6 +42,7 @@ class MountType(Enum): key: Optional[str] = None group_version: Optional[str] = None mount_requirement: MountType = MountType.ANY + env_var: Optional[str] = None def __post_init__(self): from flytekit.configuration.plugin import get_plugin @@ -56,6 +60,7 @@ def to_flyte_idl(self) -> _sec.Secret: group_version=self.group_version, key=self.key, mount_requirement=self.mount_requirement.value, + env_var=self.env_var, ) @classmethod @@ -65,6 +70,7 @@ def from_flyte_idl(cls, pb2_object: _sec.Secret) -> "Secret": group_version=pb2_object.group_version if pb2_object.group_version else None, key=pb2_object.key if pb2_object.key else None, mount_requirement=Secret.MountType(pb2_object.mount_requirement), + env_var=pb2_object.env_var if pb2_object.env_var else None, ) diff --git a/pyproject.toml b/pyproject.toml index 63a2dce6a9..765d66e5e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "diskcache>=5.2.1", "docker>=4.0.0", "docstring-parser>=0.9.0", - "flyteidl>=1.14.1", + "flyteidl>=1.14.2", "fsspec>=2023.3.0", "gcsfs>=2023.3.0", "googleapis-common-protos>=1.57", diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index cef154b6d8..39ff09f169 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -1,4 +1,5 @@ import botocore.session +import shutil from contextlib import ExitStack, contextmanager import datetime import hashlib @@ -908,3 +909,49 @@ def retry_operation(operation): remote.wait(execution=execution, timeout=datetime.timedelta(minutes=5)) assert execution.outputs["o0"] == {"title": "my report", "data": [1.0, 2.0, 3.0, 4.0, 5.0]} + + +@pytest.fixture +def kubectl_secret(): + secret = "abc-xyz" + # Create secret + kubectl = shutil.which("kubectl") + if kubectl is None: + pytest.skip("kubectl not found") + + subprocess.run([ + kubectl, + "create", + "secret", + "-n", + "flytesnacks-development", + "generic", + "my-group", + f"--from-literal=token={secret}", + ], capture_output=True, text=True) + yield secret + + # Remove secret + subprocess.run([ + kubectl, + "delete", + "secrets", + "-n", + "flytesnacks-development", + "my-group", + ], capture_output=True, text=True) + + +# To enable this test, kubectl must be available. +@pytest.mark.skip(reason="Waiting for flyte release that includes https://github.com/flyteorg/flyte/pull/6176") +@pytest.mark.parametrize("task", ["get_secret_env_var", "get_secret_file"]) +def test_check_secret(kubectl_secret, task): + execution_id = run("get_secret.py", task) + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + execution = remote.fetch_execution(name=execution_id) + execution = remote.wait(execution=execution) + assert execution.closure.phase == WorkflowExecutionPhase.SUCCEEDED, ( + f"Execution failed with phase: {execution.closure.phase}" + ) + assert execution.outputs['o0'] == kubectl_secret diff --git a/tests/flytekit/integration/remote/workflows/basic/get_secret.py b/tests/flytekit/integration/remote/workflows/basic/get_secret.py new file mode 100644 index 0000000000..a7d7ecb488 --- /dev/null +++ b/tests/flytekit/integration/remote/workflows/basic/get_secret.py @@ -0,0 +1,26 @@ +from flytekit import task, Secret, workflow +from os import getenv + +secret_env_var = Secret( + group="my-group", + key="token", + env_var="MY_SECRET", + mount_requirement=Secret.MountType.ENV_VAR, +) +secret_env_file = Secret( + group="my-group", + key="token", + env_var="MY_SECRET_FILE", + mount_requirement=Secret.MountType.FILE, +) + + +@task(secret_requests=[secret_env_var]) +def get_secret_env_var() -> str: + return getenv("MY_SECRET", "") + + +@task(secret_requests=[secret_env_file]) +def get_secret_file() -> str: + with open(getenv("MY_SECRET_FILE")) as f: + return f.read() From 88b651cb26a20989b52740f20f11e53e3c7265ea Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 23 Jan 2025 16:19:38 -0800 Subject: [PATCH 19/27] Fix async coroutine limit not respected and add s3/gcs chunk size (#3080) Signed-off-by: Yee Hing Tong --- flytekit/core/data_persistence.py | 29 +++++++ flytekit/core/type_engine.py | 31 +++---- .../unit/core/test_data_persistence.py | 34 +++++++- tests/flytekit/unit/core/test_list.py | 85 +++++++++++++++++++ 4 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 tests/flytekit/unit/core/test_list.py diff --git a/flytekit/core/data_persistence.py b/flytekit/core/data_persistence.py index 0640bc2eb5..25d1ddc688 100644 --- a/flytekit/core/data_persistence.py +++ b/flytekit/core/data_persistence.py @@ -52,6 +52,10 @@ Uploadable = typing.Union[str, os.PathLike, pathlib.Path, bytes, io.BufferedReader, io.BytesIO, io.StringIO] +# This is the default chunk size flytekit will use for writing to S3 and GCS. This is set to 25MB by default and is +# configurable by the user if needed. This is used when put() is called on filesystems. +_WRITE_SIZE_CHUNK_BYTES = int(os.environ.get("_F_P_WRITE_CHUNK_SIZE", "26214400")) # 25 * 2**20 + def s3_setup_args(s3_cfg: configuration.S3Config, anonymous: bool = False) -> Dict[str, Any]: kwargs: Dict[str, Any] = { @@ -108,6 +112,27 @@ def get_fsspec_storage_options( return {} +def get_additional_fsspec_call_kwargs(protocol: typing.Union[str, tuple], method_name: str) -> Dict[str, Any]: + """ + These are different from the setup args functions defined above. Those kwargs are applied when asking fsspec + to create the filesystem. These kwargs returned here are for when the filesystem's methods are invoked. + + :param protocol: s3, gcs, etc. + :param method_name: Pass in the __name__ of the fsspec.filesystem function. _'s will be ignored. + """ + kwargs = {} + method_name = method_name.replace("_", "") + if isinstance(protocol, tuple): + protocol = protocol[0] + + # For s3fs and gcsfs, we feel the default chunksize of 50MB is too big. + # Re-evaluate these kwargs when we move off of s3fs to obstore. + if method_name == "put" and protocol in ["s3", "gs"]: + kwargs["chunksize"] = _WRITE_SIZE_CHUNK_BYTES + + return kwargs + + @decorator def retry_request(func, *args, **kwargs): # TODO: Remove this method once s3fs has a new release. https://github.com/fsspec/s3fs/pull/865 @@ -353,6 +378,10 @@ async def _put(self, from_path: str, to_path: str, recursive: bool = False, **kw if "metadata" not in kwargs: kwargs["metadata"] = {} kwargs["metadata"].update(self._execution_metadata) + + additional_kwargs = get_additional_fsspec_call_kwargs(file_system.protocol, file_system.put.__name__) + kwargs.update(additional_kwargs) + if isinstance(file_system, AsyncFileSystem): dst = await file_system._put(from_path, to_path, recursive=recursive, **kwargs) # pylint: disable=W0212 else: diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 026921a205..641e7b5192 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -60,6 +60,8 @@ DEFINITIONS = "definitions" TITLE = "title" +_TYPE_ENGINE_COROS_BATCH_SIZE = int(os.environ.get("_F_TE_MAX_COROS", "10")) + # In Mashumaro, the default encoder uses strict_map_key=False, while the default decoder uses strict_map_key=True. # This is relevant for cases like Dict[int, str]. @@ -1686,10 +1688,9 @@ async def async_to_literal( raise TypeTransformerFailedError("Expected a list") t = self.get_sub_type(python_type) - lit_list = [ - asyncio.create_task(TypeEngine.async_to_literal(ctx, x, t, expected.collection_type)) for x in python_val - ] - lit_list = await _run_coros_in_chunks(lit_list) + lit_list = [TypeEngine.async_to_literal(ctx, x, t, expected.collection_type) for x in python_val] + + lit_list = await _run_coros_in_chunks(lit_list, batch_size=_TYPE_ENGINE_COROS_BATCH_SIZE) return Literal(collection=LiteralCollection(literals=lit_list)) @@ -1711,7 +1712,7 @@ async def async_to_python_value( # type: ignore st = self.get_sub_type(expected_python_type) result = [TypeEngine.async_to_python_value(ctx, x, st) for x in lits] - result = await _run_coros_in_chunks(result) + result = await _run_coros_in_chunks(result, batch_size=_TYPE_ENGINE_COROS_BATCH_SIZE) return result # type: ignore # should be a list, thinks its a tuple def guess_python_type(self, literal_type: LiteralType) -> list: # type: ignore @@ -2158,13 +2159,10 @@ async def async_to_literal( else: _, v_type = self.extract_types_or_metadata(python_type) - lit_map[k] = asyncio.create_task( - TypeEngine.async_to_literal(ctx, v, cast(type, v_type), expected.map_value_type) - ) - - await _run_coros_in_chunks([c for c in lit_map.values()]) - for k, v in lit_map.items(): - lit_map[k] = v.result() + lit_map[k] = TypeEngine.async_to_literal(ctx, v, cast(type, v_type), expected.map_value_type) + vals = await _run_coros_in_chunks([c for c in lit_map.values()], batch_size=_TYPE_ENGINE_COROS_BATCH_SIZE) + for idx, k in zip(range(len(vals)), lit_map.keys()): + lit_map[k] = vals[idx] return Literal(map=LiteralMap(literals=lit_map)) @@ -2185,12 +2183,11 @@ async def async_to_python_value(self, ctx: FlyteContext, lv: Literal, expected_p raise TypeError("TypeMismatch. Destination dictionary does not accept 'str' key") py_map = {} for k, v in lv.map.literals.items(): - fut = asyncio.create_task(TypeEngine.async_to_python_value(ctx, v, cast(Type, tp[1]))) - py_map[k] = fut + py_map[k] = TypeEngine.async_to_python_value(ctx, v, cast(Type, tp[1])) - await _run_coros_in_chunks([c for c in py_map.values()]) - for k, v in py_map.items(): - py_map[k] = v.result() + vals = await _run_coros_in_chunks([c for c in py_map.values()], batch_size=_TYPE_ENGINE_COROS_BATCH_SIZE) + for idx, k in zip(range(len(vals)), py_map.keys()): + py_map[k] = vals[idx] return py_map diff --git a/tests/flytekit/unit/core/test_data_persistence.py b/tests/flytekit/unit/core/test_data_persistence.py index 116717b92d..ab56f5d07d 100644 --- a/tests/flytekit/unit/core/test_data_persistence.py +++ b/tests/flytekit/unit/core/test_data_persistence.py @@ -10,9 +10,10 @@ import mock import pytest from azure.identity import ClientSecretCredential, DefaultAzureCredential +from mock import AsyncMock from flytekit.configuration import Config -from flytekit.core.data_persistence import FileAccessProvider +from flytekit.core.data_persistence import FileAccessProvider, get_additional_fsspec_call_kwargs from flytekit.core.local_fsspec import FlyteLocalFileSystem @@ -210,6 +211,37 @@ def __init__(self, *args, **kwargs): fp.get_filesystem("testgetfs", test_arg="test_arg") +def test_get_additional_fsspec_call_kwargs(): + with mock.patch("flytekit.core.data_persistence._WRITE_SIZE_CHUNK_BYTES", 12345): + kwargs = get_additional_fsspec_call_kwargs(("s3", "s3a"), "put") + assert kwargs == {"chunksize": 12345} + + kwargs = get_additional_fsspec_call_kwargs("s3", "_put") + assert kwargs == {"chunksize": 12345} + + kwargs = get_additional_fsspec_call_kwargs("s3", "get") + assert kwargs == {} + + +@pytest.mark.asyncio +@mock.patch("flytekit.core.data_persistence.FileAccessProvider.get_async_filesystem_for_path", new_callable=AsyncMock) +@mock.patch("flytekit.core.data_persistence.get_additional_fsspec_call_kwargs") +async def test_chunk_size(mock_call_kwargs, mock_get_fs): + mock_call_kwargs.return_value = {"chunksize": 1234} + mock_fs = mock.MagicMock() + mock_get_fs.return_value = mock_fs + + mock_fs.protocol = ("s3", "s3a") + fp = FileAccessProvider("/tmp", "s3://container/path/within/container") + + def put(*args, **kwargs): + assert "chunksize" in kwargs + + mock_fs.put = put + upload_location = await fp._put("/tmp/foo", "s3://bar") + assert upload_location == "s3://bar" + + @pytest.mark.sandbox_test def test_put_raw_data_bytes(): dc = Config.for_sandbox().data_config diff --git a/tests/flytekit/unit/core/test_list.py b/tests/flytekit/unit/core/test_list.py new file mode 100644 index 0000000000..96ee2efe78 --- /dev/null +++ b/tests/flytekit/unit/core/test_list.py @@ -0,0 +1,85 @@ +import asyncio +import typing + +import mock +import pytest + +from flytekit.core.context_manager import FlyteContext +from flytekit.core.type_engine import ( + AsyncTypeTransformer, + TypeEngine, +) +from flytekit.models.literals import ( + Literal, + Primitive, + Scalar, +) +from flytekit.models.types import LiteralType, SimpleType + + +class MyInt: + def __init__(self, x: int): + self.val = x + + def __eq__(self, other): + if not isinstance(other, MyInt): + return False + return other.val == self.val + + +class MyIntAsyncTransformer(AsyncTypeTransformer[MyInt]): + def __init__(self): + super().__init__(name="MyAsyncInt", t=MyInt) + self.my_lock = asyncio.Lock() + self.my_count = 0 + + def assert_type(self, t, v): + return + + def get_literal_type(self, t: typing.Type[MyInt]) -> LiteralType: + return LiteralType(simple=SimpleType.INTEGER) + + async def async_to_literal( + self, + ctx: FlyteContext, + python_val: MyInt, + python_type: typing.Type[MyInt], + expected: LiteralType, + ) -> Literal: + async with self.my_lock: + self.my_count += 1 + if self.my_count > 2: + raise ValueError("coroutine count exceeded") + await asyncio.sleep(0.1) + lit = Literal(scalar=Scalar(primitive=Primitive(integer=python_val.val))) + + async with self.my_lock: + self.my_count -= 1 + + return lit + + async def async_to_python_value( + self, ctx: FlyteContext, lv: Literal, expected_python_type: typing.Type[MyInt] + ) -> MyInt: + return MyInt(lv.scalar.primitive.integer) + + def guess_python_type(self, literal_type: LiteralType) -> typing.Type[MyInt]: + return MyInt + + +@pytest.mark.asyncio +async def test_coroutine_batching_of_list_transformer(): + TypeEngine.register(MyIntAsyncTransformer()) + + lt = LiteralType(simple=SimpleType.INTEGER) + python_val = [MyInt(10), MyInt(11), MyInt(12), MyInt(13), MyInt(14)] + ctx = FlyteContext.current_context() + + with mock.patch("flytekit.core.type_engine._TYPE_ENGINE_COROS_BATCH_SIZE", 2): + TypeEngine.to_literal(ctx, python_val, typing.List[MyInt], lt) + + with mock.patch("flytekit.core.type_engine._TYPE_ENGINE_COROS_BATCH_SIZE", 5): + with pytest.raises(ValueError): + TypeEngine.to_literal(ctx, python_val, typing.List[MyInt], lt) + + del TypeEngine._REGISTRY[MyInt] From 32a50c1da1a98a70bab05fab0bc0b4199bcffc3c Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 24 Jan 2025 16:40:41 -0800 Subject: [PATCH 20/27] Structured datasets in containers (dc/pydantic) (#3086) Signed-off-by: Yee Hing Tong --- flytekit/core/type_engine.py | 2 +- flytekit/remote/remote_fs.py | 6 ++++ .../types/structured/structured_dataset.py | 12 +++++-- tests/flytekit/unit/core/test_dataclass.py | 34 +++++++++++++++++- .../test_pydantic_basemodel_transformer.py | 34 ++++++++++++++++-- .../test_structured_dataset.py | 36 +++++++++++++++++-- 6 files changed, 114 insertions(+), 10 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 641e7b5192..288b48077a 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -1361,7 +1361,7 @@ def to_literal( ) -> Literal: """ The current dance is because we are allowing users to call from an async function, this synchronous - to_literal function, and allowing this to_literal function, to then invoke yet another async functionl, + to_literal function, and allowing this to_literal function, to then invoke yet another async function, namely an async transformer. """ from flytekit.core.promise import Promise diff --git a/flytekit/remote/remote_fs.py b/flytekit/remote/remote_fs.py index c85d93959f..ace6ebd852 100644 --- a/flytekit/remote/remote_fs.py +++ b/flytekit/remote/remote_fs.py @@ -65,6 +65,8 @@ def _upload_chunk(self, final=False): """Only uploads the file at once from the buffer. Not suitable for large files as the buffer will blow the memory for very large files. Suitable for default values or local dataframes being uploaded all at once. + + This function is called by fsspec.flush(). This will create a new file upload location. """ if final is False: return False @@ -72,6 +74,10 @@ def _upload_chunk(self, final=False): data = self.buffer.read() try: + # The inputs here are flipped a bit, it should be the filename is set to the filename and the filename root + # is something deterministic, like a hash. But since this is supposed to mimic open(), we can't hash. + # With the args currently below, the backend will create a random suffix for the filename. + # Since no hash is set on it, we will not be able to write to it again (which is totally fine). res = self._remote.client.get_upload_signed_url( self._remote.default_project, self._remote.default_domain, diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index da9cc79753..4812ce0856 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -22,7 +22,7 @@ from flytekit import lazy_module from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError +from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError, modify_literal_uris from flytekit.deck.renderer import Renderable from flytekit.extras.pydantic_transformer.decorator import model_serializer, model_validator from flytekit.loggers import developer_logger, logger @@ -63,6 +63,7 @@ class (that is just a model, a Python class representation of the protobuf). file_format: typing.Optional[str] = field(default=GENERIC_FORMAT, metadata=config(mm_field=fields.String())) def _serialize(self) -> Dict[str, Optional[str]]: + # dataclass case lv = StructuredDatasetTransformerEngine().to_literal( FlyteContextManager.current_context(), self, type(self), None ) @@ -85,7 +86,7 @@ def _deserialize(cls, value) -> "StructuredDataset": FlyteContextManager.current_context(), Literal( scalar=Scalar( - structured_dataset=StructuredDataset( + structured_dataset=literals.StructuredDataset( metadata=StructuredDatasetMetadata( structured_dataset_type=StructuredDatasetType(format=file_format) ), @@ -98,6 +99,7 @@ def _deserialize(cls, value) -> "StructuredDataset": @model_serializer def serialize_structured_dataset(self) -> Dict[str, Optional[str]]: + # pydantic case lv = StructuredDatasetTransformerEngine().to_literal( FlyteContextManager.current_context(), self, type(self), None ) @@ -117,7 +119,7 @@ def deserialize_structured_dataset(self, info) -> StructuredDataset: FlyteContextManager.current_context(), Literal( scalar=Scalar( - structured_dataset=StructuredDataset( + structured_dataset=literals.StructuredDataset( metadata=StructuredDatasetMetadata( structured_dataset_type=StructuredDatasetType(format=self.file_format) ), @@ -807,6 +809,10 @@ def encode( # with a format of "" is used. sd_model.metadata._structured_dataset_type.format = handler.supported_format lit = Literal(scalar=Scalar(structured_dataset=sd_model)) + + # Because the handler.encode may have uploaded something, and because the sd may end up living inside a + # dataclass, we need to modify any uploaded flyte:// urls here. + modify_literal_uris(lit) sd._literal_sd = sd_model sd._already_uploaded = True return lit diff --git a/tests/flytekit/unit/core/test_dataclass.py b/tests/flytekit/unit/core/test_dataclass.py index 09a6d26f54..ea613bd8b7 100644 --- a/tests/flytekit/unit/core/test_dataclass.py +++ b/tests/flytekit/unit/core/test_dataclass.py @@ -1,6 +1,7 @@ import pytest from enum import Enum -from dataclasses_json import DataClassJsonMixin +import mock +from pathlib import Path from mashumaro.mixins.json import DataClassJSONMixin import os import sys @@ -18,6 +19,8 @@ from flytekit.types.file import FlyteFile from flytekit.types.structured import StructuredDataset +pd = pytest.importorskip("pandas") + @pytest.fixture def local_dummy_txt_file(): @@ -1175,3 +1178,32 @@ class B(): res = DataclassTransformer()._make_dataclass_serializable(b, Union[None, A, B]) assert res.x.path == "s3://my-bucket/my-file" + + +@mock.patch("flytekit.remote.remote_fs.FlytePathResolver") +def test_modify_literal_uris_call(mock_resolver): + ctx = FlyteContextManager.current_context() + + sd = StructuredDataset(dataframe=pd.DataFrame( + {"a": [1, 2], "b": [3, 4]})) + + @dataclass + class DC1: + s: StructuredDataset + + bm = DC1(s=sd) + + def mock_resolve_remote_path(flyte_uri: str): + p = Path(flyte_uri) + if p.exists(): + return "/my/replaced/val" + return "" + + mock_resolver.resolve_remote_path.side_effect = mock_resolve_remote_path + mock_resolver.protocol = "/" + + lt = TypeEngine.to_literal_type(DC1) + lit = TypeEngine.to_literal(ctx, bm, DC1, lt) + + bm_revived = TypeEngine.to_python_value(ctx, lit, DC1) + assert bm_revived.s.literal.uri == "/my/replaced/val" diff --git a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py index d929e4d4fa..63127eaee2 100644 --- a/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py +++ b/tests/flytekit/unit/extras/pydantic_transformer/test_pydantic_basemodel_transformer.py @@ -1,9 +1,10 @@ import os import tempfile from enum import Enum +from pathlib import Path from typing import Dict, List, Optional, Union from unittest.mock import patch - +import mock import pytest from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -15,12 +16,13 @@ from flytekit.core.type_engine import TypeEngine from flytekit.models.annotation import TypeAnnotation from flytekit.models.literals import Literal, Scalar -from flytekit.models.types import LiteralType, SimpleType from flytekit.types.directory import FlyteDirectory from flytekit.types.file import FlyteFile from flytekit.types.schema import FlyteSchema from flytekit.types.structured import StructuredDataset +pd = pytest.importorskip("pandas") + class Status(Enum): PENDING = "pending" @@ -992,3 +994,31 @@ class BM(BaseModel): c: str = "Hello, Flyte" assert TypeEngine.to_literal_type(BM).annotation == TypeAnnotation({CACHE_KEY_METADATA: {SERIALIZATION_FORMAT: MESSAGEPACK}}) + + +@mock.patch("flytekit.remote.remote_fs.FlytePathResolver") +def test_modify_literal_uris_call(mock_resolver): + ctx = FlyteContextManager.current_context() + + sd = StructuredDataset(dataframe=pd.DataFrame( + {"a": [1, 2], "b": [3, 4]})) + + class BM(BaseModel): + s: StructuredDataset + + bm = BM(s=sd) + + def mock_resolve_remote_path(flyte_uri: str): + p = Path(flyte_uri) + if p.exists(): + return "/my/replaced/val" + return "" + + mock_resolver.resolve_remote_path.side_effect = mock_resolve_remote_path + mock_resolver.protocol = "/" + + lt = TypeEngine.to_literal_type(BM) + lit = TypeEngine.to_literal(ctx, bm, BM, lt) + + bm_revived = TypeEngine.to_python_value(ctx, lit, BM) + assert bm_revived.s.literal.uri == "/my/replaced/val" diff --git a/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py b/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py index c8efbe272d..ee535697a3 100644 --- a/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py +++ b/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py @@ -3,7 +3,7 @@ import typing from collections import OrderedDict from pathlib import Path - +import mock import google.cloud.bigquery import pytest from fsspec.utils import get_protocol @@ -661,7 +661,6 @@ def wf_with_input() -> pd.DataFrame: pd.testing.assert_frame_equal(wf_with_input(), input_val) - def test_read_sd_from_local_uri(local_tmp_pqt_file): @task @@ -677,9 +676,40 @@ def read_sd_from_local_uri(uri: str) -> pd.DataFrame: return df - df = generate_pandas() # Read sd from local uri df_local = read_sd_from_local_uri(uri=local_tmp_pqt_file) pd.testing.assert_frame_equal(df, df_local) + + +@mock.patch("flytekit.remote.remote_fs.FlytePathResolver") +@mock.patch("flytekit.types.structured.structured_dataset.StructuredDatasetTransformerEngine.get_encoder") +def test_modify_literal_uris_call(mock_get_encoder, mock_resolver): + + ctx = FlyteContextManager.current_context() + + sd = StructuredDataset(dataframe=pd.DataFrame( + {"a": [1, 2], "b": [3, 4]}), uri="bq://blah", file_format="bq") + + def mock_resolve_remote_path(flyte_uri: str) -> typing.Optional[str]: + if flyte_uri == "bq://blah": + return "bq://blah/blah/blah" + return "" + + mock_resolver.resolve_remote_path.side_effect = mock_resolve_remote_path + mock_resolver.protocol = "bq" + + dummy_encoder = mock.MagicMock() + sd_model = literals.StructuredDataset(uri="bq://blah", metadata=StructuredDatasetMetadata(StructuredDatasetType(format="parquet"))) + dummy_encoder.encode.return_value = sd_model + + mock_get_encoder.return_value = dummy_encoder + + sdte = StructuredDatasetTransformerEngine() + lt = LiteralType( + structured_dataset_type=StructuredDatasetType() + ) + + lit = sdte.encode(ctx, sd, df_type=pd.DataFrame, protocol="bq", format="parquet", structured_literal_type=lt) + assert lit.scalar.structured_dataset.uri == "bq://blah/blah/blah" From 5be0a6cedc66647aec7ade6fb9d1839465789944 Mon Sep 17 00:00:00 2001 From: Akinori Mitani Date: Fri, 24 Jan 2025 17:03:14 -0800 Subject: [PATCH 21/27] Add pip_extra_args (#3081) Signed-off-by: amitani --- flytekit/image_spec/default_builder.py | 3 +++ flytekit/image_spec/image_spec.py | 2 ++ .../flytekit/unit/core/image_spec/test_default_builder.py | 7 +++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py index 2105a49acc..df85d48b3c 100644 --- a/flytekit/image_spec/default_builder.py +++ b/flytekit/image_spec/default_builder.py @@ -216,6 +216,9 @@ def prepare_python_install(image_spec: ImageSpec, tmp_dir: Path) -> str: extra_urls = [f"--extra-index-url {url}" for url in image_spec.pip_extra_index_url] pip_install_args.extend(extra_urls) + if image_spec.pip_extra_args: + pip_install_args.append(image_spec.pip_extra_args) + requirements = [] if image_spec.requirements: requirement_basename = os.path.basename(image_spec.requirements) diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 1b9e6403e1..397ef47707 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -48,6 +48,7 @@ class ImageSpec: platform: Specify the target platforms for the build output (for example, windows/amd64 or linux/amd64,darwin/arm64 pip_index: Specify the custom pip index url pip_extra_index_url: Specify one or more pip index urls as a list + pip_extra_args: Specify one or more extra pip install arguments as a space-delimited string registry_config: Specify the path to a JSON registry config file entrypoint: List of strings to overwrite the entrypoint of the base image with, set to [] to remove the entrypoint. commands: Command to run during the building process @@ -82,6 +83,7 @@ class ImageSpec: platform: str = "linux/amd64" pip_index: Optional[str] = None pip_extra_index_url: Optional[List[str]] = None + pip_extra_args: Optional[str] = None registry_config: Optional[str] = None entrypoint: Optional[List[str]] = None commands: Optional[List[str]] = None diff --git a/tests/flytekit/unit/core/image_spec/test_default_builder.py b/tests/flytekit/unit/core/image_spec/test_default_builder.py index aadcac0a16..a50b567230 100644 --- a/tests/flytekit/unit/core/image_spec/test_default_builder.py +++ b/tests/flytekit/unit/core/image_spec/test_default_builder.py @@ -235,6 +235,7 @@ def test_create_docker_context_uv_lock(tmp_path): requirements=os.fspath(uv_lock_file), pip_index="https://url.com", pip_extra_index_url=["https://extra-url.com"], + pip_extra_args="--no-install-package library-to-skip", ) warning_msg = "uv.lock support is experimental" @@ -247,7 +248,8 @@ def test_create_docker_context_uv_lock(tmp_path): assert ( "uv sync --index-url https://url.com --extra-index-url " - "https://extra-url.com --locked --no-dev --no-install-project" + "https://extra-url.com --no-install-package library-to-skip " + "--locked --no-dev --no-install-project" ) in dockerfile_content @@ -309,6 +311,7 @@ def test_create_poetry_lock(tmp_path): name="FLYTEKIT", python_version="3.12", requirements=os.fspath(poetry_lock), + pip_extra_args="--no-directory", ) create_docker_context(image_spec, docker_context_path) @@ -317,7 +320,7 @@ def test_create_poetry_lock(tmp_path): assert dockerfile_path.exists() dockerfile_content = dockerfile_path.read_text() - assert "poetry install --no-root" in dockerfile_content + assert "poetry install --no-directory --no-root" in dockerfile_content def test_python_exec(tmp_path): From 7bfcffc016c690dbb2a08a4e47fccfa2693fc284 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Fri, 24 Jan 2025 20:05:33 -0500 Subject: [PATCH 22/27] Enable pydoclint (#3077) * Enable pydoclint Signed-off-by: Eduardo Apolinario * Regenerate baseline Signed-off-by: Eduardo Apolinario * Regenerate baseline for real this time Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .pre-commit-config.yaml | 11 +- pydoclint-errors-baseline.txt | 618 ++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 628 insertions(+), 3 deletions(-) create mode 100644 pydoclint-errors-baseline.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea236cc92f..a4e2c6e913 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,13 @@ repos: rev: v2.3.0 hooks: - id: codespell + additional_dependencies: + - tomli + - repo: https://github.com/jsh9/pydoclint + rev: 0.6.0 + hooks: + - id: pydoclint args: - - --ignore-words-list=assertIn # Ignore 'assertIn' - additional_dependencies: [tomli] + - --style=google + - --exclude='.git|tests/flytekit/*|tests/' + - --baseline=pydoclint-errors-baseline.txt diff --git a/pydoclint-errors-baseline.txt b/pydoclint-errors-baseline.txt new file mode 100644 index 0000000000..1ac90bc20c --- /dev/null +++ b/pydoclint-errors-baseline.txt @@ -0,0 +1,618 @@ +flytekit/clients/auth/auth_client.py + DOC301: Class `AuthorizationClient`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/clients/auth/authenticator.py + DOC301: Class `PKCEAuthenticator`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/clients/raw.py + DOC301: Class `RawSynchronousFlyteClient`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/configuration/__init__.py + DOC605: Class `Image`: Attribute names match, but type hints in these attributes do not match: tag, digest (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC101: Function `_parse_image_identifier`: Docstring contains fewer arguments than in function signature. + DOC103: Function `_parse_image_identifier`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [image_identifier: str]. + DOC203: Function `_parse_image_identifier` return type(s) in docstring not consistent with the return annotation. Return annotation types: ['typing.Tuple[str, Optional[str], Optional[str]]']; docstring return section types: ['Tuple[str, str, str]'] + DOC605: Class `ImageConfig`: Attribute names match, but type hints in these attributes do not match: images (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `Config`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Config`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [data_config: DataConfig, local_sandbox_path: str, platform: PlatformConfig, secrets: SecretsConfig, stats: StatsConfig]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `SerializationSettings`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [git_repo: Optional[str]]. Arguments in the docstring but not in the actual class attributes: [entrypoint_settings: Optional[EntrypointSettings]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/configuration/file.py + DOC601: Class `LegacyConfigEntry`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `LegacyConfigEntry`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [option: str, section: str, type_: typing.Type]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `YamlConfigEntry`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `YamlConfigEntry`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [config_value_type: typing.Type, switch: str]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC301: Class `ConfigFile`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/array_node.py + DOC301: Class `ArrayNode`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/array_node_map_task.py + DOC301: Class `ArrayNodeMapTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/artifact.py + DOC301: Class `Artifact`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/base_sql_task.py + DOC301: Class `SQLTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/base_task.py + DOC601: Class `TaskMetadata`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `TaskMetadata`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [cache: bool, cache_ignore_input_vars: Tuple[str, ...], cache_serialize: bool, cache_version: str, deprecated: str, interruptible: Optional[bool], is_eager: bool, pod_template_name: Optional[str], retries: int, timeout: Optional[Union[datetime.timedelta, int]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC301: Class `PythonTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC001: Function/method `post_execute`: Potential formatting errors in docstring. Error message: Expected a colon in 'rval is returned value from call to execute'. (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.) + DOC101: Method `PythonTask.post_execute`: Docstring contains fewer arguments than in function signature. + DOC103: Method `PythonTask.post_execute`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [rval: Any, user_params: Optional[ExecutionParameters]]. + DOC201: Method `PythonTask.post_execute` does not have a return section in docstring + DOC203: Method `PythonTask.post_execute` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). +-------------------- +flytekit/core/checkpointer.py + DOC109: Method `Checkpoint.save`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC110: Method `Checkpoint.save`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints + DOC105: Method `Checkpoint.save`: Argument names match, but type hints in these args do not match: cp + DOC501: Method `Checkpoint.save` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Method `Checkpoint.save` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['NotImplementedError']. + DOC301: Class `SyncCheckpoint`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/container_task.py + DOC001: Function/method `_prepare_command_and_volumes`: Potential formatting errors in docstring. Error message: No specification for "Parameters": "" (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.) + DOC101: Method `ContainerTask._prepare_command_and_volumes`: Docstring contains fewer arguments than in function signature. + DOC103: Method `ContainerTask._prepare_command_and_volumes`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: , cmd_and_args: List[str]]. + DOC201: Method `ContainerTask._prepare_command_and_volumes` does not have a return section in docstring + DOC203: Method `ContainerTask._prepare_command_and_volumes` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). +-------------------- +flytekit/core/context_manager.py + DOC301: Class `ExecutionParameters`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC605: Class `CompilationState`: Attribute names match, but type hints in these attributes do not match: nodes (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ExecutionState`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [branch_eval_mode: Optional[BranchEvalMode], user_space_params: Optional[ExecutionParameters]]. Arguments in the docstring but not in the actual class attributes: [branch_eval_mode Optional[BranchEvalMode]: , user_space_params Optional[ExecutionParameters]: ]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC101: Method `ExecutionState.__init__`: Docstring contains fewer arguments than in function signature. + DOC103: Method `ExecutionState.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [branch_eval_mode: Optional[BranchEvalMode], engine_dir: Optional[Union[os.PathLike, str]], mode: Optional[ExecutionState.Mode], user_space_params: Optional[ExecutionParameters], working_dir: Union[os.PathLike, str]]. + DOC501: Method `ExecutionState.__init__` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Method `ExecutionState.__init__` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['ValueError']. + DOC603: Class `OutputMetadataTracker`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [output_metadata: typing.Dict[typing.Any, OutputMetadata]]. Arguments in the docstring but not in the actual class attributes: [output_metadata Optional[TaskOutputMetadata]: ]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/core/data_persistence.py + DOC301: Class `FileAccessProvider`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/interface.py + DOC301: Class `Interface`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/legacy_map_task.py + DOC301: Class `MapPythonTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/mock_stats.py + DOC301: Class `MockStats`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `_Timer`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/notification.py + DOC301: Class `Notification`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `PagerDuty`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Email`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Slack`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/options.py + DOC601: Class `Options`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Options`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [annotations: typing.Optional[common_models.Annotations], disable_notifications: typing.Optional[bool], labels: typing.Optional[common_models.Labels], max_parallelism: typing.Optional[int], notifications: typing.Optional[typing.List[common_models.Notification]], overwrite_cache: typing.Optional[bool], raw_output_data_config: typing.Optional[common_models.RawOutputDataConfig], security_context: typing.Optional[security.SecurityContext]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/core/promise.py + DOC301: Class `NodeOutput`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/python_auto_container.py + DOC301: Class `PythonAutoContainerTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC605: Class `PickledEntityMetadata`: Attribute names match, but type hints in these attributes do not match: python_version (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC605: Class `PickledEntity`: Attribute names match, but type hints in these attributes do not match: metadata, entities (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/core/python_customized_container_task.py + DOC301: Class `PythonCustomizedContainerTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/python_function_task.py + DOC301: Class `PythonInstanceTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `PythonFunctionTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/reference_entity.py + DOC301: Class `ReferenceTemplate`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ReferenceSpec`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/schedule.py + DOC301: Class `CronSchedule`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `FixedRate`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `OnSchedule`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/task.py + DOC101: Function `reference_task`: Docstring contains fewer arguments than in function signature. + DOC103: Function `reference_task`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [domain: str, name: str, project: str, version: str]. + DOC201: Function `reference_task` does not have a return section in docstring + DOC203: Function `reference_task` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). + DOC301: Class `Echo`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/type_engine.py + DOC301: Class `LiteralsResolver`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/utils.py + DOC301: Class `Directory`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `AutoDeletingTempDir`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `timeit`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ClassDecorator`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/core/workflow.py + DOC101: Function `workflow`: Docstring contains fewer arguments than in function signature. + DOC103: Function `workflow`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [_workflow_function: Optional[Callable[P, FuncOut]], default_options: Optional[Options], docs: Optional[Documentation], failure_policy: Optional[WorkflowFailurePolicy], interruptible: bool, on_failure: Optional[Union[WorkflowBase, Task]], pickle_untyped: bool]. + DOC201: Function `workflow` does not have a return section in docstring + DOC203: Function `workflow` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). + DOC101: Function `reference_workflow`: Docstring contains fewer arguments than in function signature. + DOC103: Function `reference_workflow`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [domain: str, name: str, project: str, version: str]. + DOC201: Function `reference_workflow` does not have a return section in docstring + DOC203: Function `reference_workflow` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). +-------------------- +flytekit/exceptions/system.py + DOC301: Class `FlyteNonRecoverableSystemException`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/exceptions/user.py + DOC301: Class `FlyteUserRuntimeException`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/extend/backend/base_agent.py + DOC601: Class `Resource`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Resource`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [custom_info: Optional[typing.Dict[str, Any]], log_links: Optional[List[TaskLog]], message: Optional[str], outputs: Optional[Union[LiteralMap, typing.Dict[str, Any]]], phase: TaskExecution.Phase]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/extras/sqlite3/task.py + DOC601: Class `SQLite3Config`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `SQLite3Config`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [compressed: bool, uri: str]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/extras/tasks/shell.py + DOC601: Class `ProcessResult`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ProcessResult`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [error: str, output: str, returncode: int]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC001: Class `OutputLocation`: Potential formatting errors in docstring. Error message: No specification for "Args": "" + DOC601: Class `OutputLocation`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `OutputLocation`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [location: typing.Union[os.PathLike, str], var: str, var_type: typing.Type]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC101: Function `subproc_execute`: Docstring contains fewer arguments than in function signature. + DOC103: Function `subproc_execute`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: ]. + DOC503: Function `subproc_execute` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: ['Exception', 'Exception']. Raised exceptions in the body: ['RuntimeError']. + DOC301: Class `ShellTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `RawShellTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/extras/tensorflow/record.py + DOC601: Class `TFRecordDatasetConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `TFRecordDatasetConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [buffer_size: Optional[int], compression_type: Optional[str], name: Optional[str], num_parallel_reads: Optional[int]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/image_spec/image_spec.py + DOC601: Class `ImageSpec`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ImageSpec`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [apt_packages: Optional[List[str]], base_image: Optional[Union[str, 'ImageSpec']], builder: Optional[str], commands: Optional[List[str]], conda_channels: Optional[List[str]], conda_packages: Optional[List[str]], copy: Optional[List[str]], cuda: Optional[str], cudnn: Optional[str], entrypoint: Optional[List[str]], env: Optional[typing.Dict[str, str]], name: str, packages: Optional[List[str]], pip_extra_index_url: Optional[List[str]], pip_index: Optional[str], platform: str, python_exec: Optional[str], python_version: str, registry: Optional[str], registry_config: Optional[str], requirements: Optional[str], source_copy_mode: Optional[CopyFileDetection], source_root: Optional[str], tag_format: Optional[str]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC109: Method `ImageSpecBuilder.build_image`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC110: Method `ImageSpecBuilder.build_image`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints + DOC105: Method `ImageSpecBuilder.build_image`: Argument names match, but type hints in these args do not match: image_spec + DOC203: Method `ImageSpecBuilder.build_image` return type(s) in docstring not consistent with the return annotation. Return annotation types: ['Optional[str]']; docstring return section types: ['fully_qualified_image_name'] + DOC501: Method `ImageSpecBuilder.build_image` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Method `ImageSpecBuilder.build_image` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['NotImplementedError']. + DOC109: Method `ImageSpecBuilder.should_build`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC110: Method `ImageSpecBuilder.should_build`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints + DOC105: Method `ImageSpecBuilder.should_build`: Argument names match, but type hints in these args do not match: image_spec + DOC203: Method `ImageSpecBuilder.should_build` return type(s) in docstring not consistent with the return annotation. Return annotation types: ['bool']; docstring return section types: [''] +-------------------- +flytekit/interactive/utils.py + DOC106: Function `load_module_from_path`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Function `load_module_from_path`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC105: Function `load_module_from_path`: Argument names match, but type hints in these args do not match: module_name, path + DOC203: Function `load_module_from_path` return type(s) in docstring not consistent with the return annotation. Return annotation has 0 type(s); docstring return section has 1 type(s). + DOC106: Function `get_task_inputs`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Function `get_task_inputs`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC105: Function `get_task_inputs`: Argument names match, but type hints in these args do not match: task_module_name, task_name, context_working_dir + DOC203: Function `get_task_inputs` return type(s) in docstring not consistent with the return annotation. Return annotation has 0 type(s); docstring return section has 1 type(s). +-------------------- +flytekit/interactive/vscode_lib/config.py + DOC601: Class `VscodeConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `VscodeConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [code_server_dir_names: Optional[Dict[str, str]], code_server_remote_paths: Optional[Dict[str, str]], extension_remote_paths: Optional[List[str]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/interactive/vscode_lib/decorator.py + DOC101: Function `exit_handler`: Docstring contains fewer arguments than in function signature. + DOC107: Function `exit_handler`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC103: Function `exit_handler`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [args: , kwargs: , task_function: ]. + DOC201: Function `exit_handler` does not have a return section in docstring + DOC107: Function `download_file`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC105: Function `download_file`: Argument names match, but type hints in these args do not match: url, target_dir + DOC203: Function `download_file` return type(s) in docstring not consistent with the return annotation. Return annotation has 0 type(s); docstring return section has 1 type(s). + DOC501: Function `download_file` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Function `download_file` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['ValueError']. + DOC106: Function `prepare_interactive_python`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Function `prepare_interactive_python`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC105: Function `prepare_interactive_python`: Argument names match, but type hints in these args do not match: task_function + DOC301: Class `vscode`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/admin/common.py + DOC301: Class `Sort`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/admin/task_execution.py + DOC301: Class `TaskExecutionClosure`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskExecution`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/admin/workflow.py + DOC301: Class `WorkflowSpec`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Workflow`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `WorkflowClosure`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/array_job.py + DOC301: Class `ArrayJob`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/common.py + DOC301: Class `NamedEntityIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `EmailNotification`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SlackNotification`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `PagerDutyNotification`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Notification`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Labels`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Annotations`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `UrlBlob`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `AuthRole`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `RawOutputDataConfig`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/compiler.py + DOC301: Class `IdList`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ConnectionSet`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `CompiledWorkflow`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `CompiledTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `CompiledWorkflowClosure`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/condition.py + DOC301: Class `ComparisonExpression`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ConjunctionExpression`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Operand`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BooleanExpression`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/errors.py + DOC301: Class `ContainerError`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ErrorDocument`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/execution.py + DOC301: Class `ExecutionError`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskLog`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/identifier.py + DOC301: Class `Identifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `WorkflowExecutionIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `NodeExecutionIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskExecutionIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SignalIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/types.py + DOC301: Class `BlobType`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/core/workflow.py + DOC301: Class `IfBlock`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `IfElseBlock`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BranchNode`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `NodeMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SignalCondition`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ApproveCondition`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SleepCondition`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ArrayNode`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Node`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskNode`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `WorkflowNode`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC001: Class `OnFailurePolicy`: Potential formatting errors in docstring. Error message: Expected a colon in "FAIL_IMMEDIATELY Instructs the system to fail as soon as a node fails in the\n workflow. It'll automatically abort all currently running nodes and\n clean up resources before finally marking the workflow executions as failed.". + DOC601: Class `OnFailurePolicy`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `OnFailurePolicy`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [FAIL_AFTER_EXECUTABLE_NODES_COMPLETE: , FAIL_IMMEDIATELY: ]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC301: Class `WorkflowMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `WorkflowMetadataDefaults`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `WorkflowTemplate`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Alias`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/documentation.py + DOC601: Class `Documentation`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Documentation`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [long_description: Optional[Description], short_description: Optional[str], source_code: Optional[SourceCode]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/models/dynamic_job.py + DOC301: Class `DynamicJobSpec`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/execution.py + DOC301: Class `ExecutionMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ExecutionSpec`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ClusterAssignment`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LiteralMapBlob`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Execution`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ExecutionClosure`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `NotificationList`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `_CommonDataResponse`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/filters.py + DOC301: Class `FilterList`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Filter`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SetFilter`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/interface.py + DOC301: Class `Variable`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `VariableMap`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TypedInterface`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Parameter`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ParameterMap`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/launch_plan.py + DOC301: Class `LaunchPlanMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Auth`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LaunchPlanSpec`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LaunchPlanClosure`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LaunchPlan`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/literals.py + DOC301: Class `RetryStrategy`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Primitive`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Binary`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BlobMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Blob`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BindingDataMap`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BindingDataCollection`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `BindingData`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Binding`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Schema`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Union`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `StructuredDataset`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LiteralCollection`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LiteralMap`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Scalar`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LiteralOffloadedMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Literal`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/matchable_resource.py + DOC301: Class `ClusterResourceAttributes`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ExecutionQueueAttributes`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `ExecutionClusterLabel`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `PluginOverride`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `PluginOverrides`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `MatchingAttributes`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/named_entity.py + DOC301: Class `NamedEntityIdentifier`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `NamedEntityMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/node_execution.py + DOC301: Class `NodeExecutionClosure`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `NodeExecution`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/presto.py + DOC301: Class `PrestoQuery`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/project.py + DOC301: Class `Project`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/qubole.py + DOC301: Class `HiveQuery`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `HiveQueryCollection`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `QuboleHiveJob`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/schedule.py + DOC301: Class `FixedRate`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `CronSchedule`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Schedule`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/security.py + DOC001: Class `Secret`: Potential formatting errors in docstring. Error message: Expected a colon in 'group is the Name of the secret. For example in kubernetes secrets is the name of the secret'. + DOC601: Class `Secret`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Secret`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [env_var: Optional[str], group: Optional[str], group_version: Optional[str], key: Optional[str], mount_requirement: MountType]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +flytekit/models/task.py + DOC301: Class `ResourceEntry`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Resources`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `RuntimeMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskTemplate`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskExecutionMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskSpec`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Task`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `TaskClosure`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `CompiledTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Container`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `K8sObjectMetadata`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `K8sPod`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `Sql`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/types.py + DOC301: Class `SchemaColumn`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SchemaType`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `LiteralType`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `OutputReference`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/models/workflow_closure.py + DOC301: Class `WorkflowClosure`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/remote/remote.py + DOC301: Class `FlyteRemote`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/types/directory/types.py + DOC301: Class `FlyteDirectory`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC101: Method `FlyteDirectory.crawl`: Docstring contains fewer arguments than in function signature. + DOC103: Method `FlyteDirectory.crawl`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: , maxdepth: typing.Optional[int], topdown: bool]. + DOC402: Method `FlyteDirectory.crawl` has "yield" statements, but the docstring does not have a "Yields" section + DOC404: Method `FlyteDirectory.crawl` yield type(s) in docstring not consistent with the return annotation. Return annotation exists, but docstring "yields" section does not exist or has 0 type(s). +-------------------- +flytekit/types/file/__init__.py + DOC101: Method `FileExt.__init__`: Docstring contains fewer arguments than in function signature. + DOC103: Method `FileExt.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [ext: str]. +-------------------- +flytekit/types/file/file.py + DOC301: Class `FlyteFile`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +flytekit/types/structured/structured_dataset.py + DOC301: Class `StructuredDatasetEncoder`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `StructuredDatasetDecoder`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-async-fsspec/flytekitplugins/async_fsspec/s3fs/s3fs.py + DOC101: Method `AsyncS3FileSystem._put_file`: Docstring contains fewer arguments than in function signature. + DOC106: Method `AsyncS3FileSystem._put_file`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Method `AsyncS3FileSystem._put_file`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC103: Method `AsyncS3FileSystem._put_file`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: ]. + DOC201: Method `AsyncS3FileSystem._put_file` does not have a return section in docstring + DOC101: Method `AsyncS3FileSystem._get_file`: Docstring contains fewer arguments than in function signature. + DOC106: Method `AsyncS3FileSystem._get_file`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Method `AsyncS3FileSystem._get_file`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC103: Method `AsyncS3FileSystem._get_file`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [concurrent_download: ]. + DOC201: Method `AsyncS3FileSystem._get_file` does not have a return section in docstring +-------------------- +plugins/flytekit-aws-athena/flytekitplugins/athena/task.py + DOC301: Class `AthenaTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py + DOC301: Class `Boto3AgentMixin`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py + DOC301: Class `SageMakerModelTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerEndpointConfigTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerEndpointTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerDeleteEndpointTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerDeleteEndpointConfigTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerDeleteModelTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `SageMakerInvokeEndpointTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-bigquery/flytekitplugins/bigquery/task.py + DOC301: Class `BigQueryTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-comet-ml/flytekitplugins/comet_ml/tracking.py + DOC105: Function `comet_ml_login`: Argument names match, but type hints in these args do not match: secret, experiment_key + DOC201: Function `comet_ml_login` does not have a return section in docstring + DOC301: Class `_comet_ml_login_class`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-duckdb/flytekitplugins/duckdb/task.py + DOC301: Class `DuckDBQuery`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC203: Method `DuckDBQuery._connect_to_duckdb` return type(s) in docstring not consistent with the return annotation. Return annotation has 0 type(s); docstring return section has 1 type(s). + DOC101: Method `DuckDBQuery._execute_query`: Docstring contains fewer arguments than in function signature. + DOC109: Method `DuckDBQuery._execute_query`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC110: Method `DuckDBQuery._execute_query`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints + DOC103: Method `DuckDBQuery._execute_query`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [con: duckdb.DuckDBPyConnection]. + DOC402: Method `DuckDBQuery._execute_query` has "yield" statements, but the docstring does not have a "Yields" section + DOC404: Method `DuckDBQuery._execute_query` yield type(s) in docstring not consistent with the return annotation. Return annotation exists, but docstring "yields" section does not exist or has 0 type(s). + DOC501: Method `DuckDBQuery._execute_query` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Method `DuckDBQuery._execute_query` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['ValueError']. +-------------------- +plugins/flytekit-flyteinteractive/flytekitplugins/flyteinteractive/jupyter_lib/decorator.py + DOC105: Function `write_example_notebook`: Argument names match, but type hints in these args do not match: task_function + DOC101: Function `exit_handler`: Docstring contains fewer arguments than in function signature. + DOC107: Function `exit_handler`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC103: Function `exit_handler`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [args: , kwargs: , task_function: ]. + DOC201: Function `exit_handler` does not have a return section in docstring + DOC301: Class `jupyter`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py + DOC601: Class `GreatExpectationsFlyteConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `GreatExpectationsFlyteConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [batch_request_config: Optional[BatchRequestConfig], checkpoint_params: Optional[Dict[str, Union[str, List[str]]]], context_root_dir: str, data_asset_name: Optional[str], data_connector_name: str, datasource_name: str, expectation_suite_name: str, local_file_path: Optional[str]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py + DOC601: Class `BatchRequestConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `BatchRequestConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [batch_identifiers: Optional[Dict[str, str]], batch_spec_passthrough: Optional[Dict[str, Any]], data_connector_query: Optional[Dict[str, Any]], runtime_parameters: Optional[Dict[str, Any]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC101: Method `GreatExpectationsTask.__init__`: Docstring contains fewer arguments than in function signature. + DOC109: Method `GreatExpectationsTask.__init__`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC110: Method `GreatExpectationsTask.__init__`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints + DOC103: Method `GreatExpectationsTask.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: , outputs: Optional[Dict[str, Type]]]. +-------------------- +plugins/flytekit-hive/flytekitplugins/hive/task.py + DOC601: Class `HiveConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `HiveConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [cluster_label: str, tags: Optional[List[str]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC301: Class `HiveTask`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC301: Class `HiveSelectTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-identity-aware-proxy/flytekitplugins/identity_aware_proxy/cli.py + DOC301: Class `GCPIdentityAwareProxyAuthenticator`: __init__() should not have a docstring; please combine it with the docstring of the class + DOC201: Function `get_service_account_id_token` does not have a return section in docstring + DOC203: Function `get_service_account_id_token` return type(s) in docstring not consistent with the return annotation. Return annotation has 1 type(s); docstring return section has 0 type(s). +-------------------- +plugins/flytekit-inference/flytekitplugins/inference/nim/serve.py + DOC301: Class `NIM`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-inference/flytekitplugins/inference/ollama/serve.py + DOC301: Class `Ollama`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-inference/flytekitplugins/inference/vllm/serve.py + DOC301: Class `VLLM`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-k8sdataservice/flytekitplugins/k8sdataservice/sensor.py + DOC301: Class `CleanupSensor`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py + DOC001: Class `RunPolicy`: Potential formatting errors in docstring. Error message: Expected a colon in 'can remain active before it is terminated. Must be a positive integer. This setting applies only to pods.'. + DOC601: Class `RunPolicy`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `RunPolicy`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [active_deadline_seconds: Optional[int], backoff_limit: Optional[int], clean_pod_policy: CleanPodPolicy, ttl_seconds_after_finished: Optional[int]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `MPIJob`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `MPIJob`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [launcher: Launcher, num_launcher_replicas: Optional[int], num_workers: Optional[int], run_policy: Optional[RunPolicy], slots: int, worker: Worker]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `HorovodJob`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `HorovodJob`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [discovery_script_path: Optional[str], elastic_timeout: Optional[int], launcher: Launcher, log_level: Optional[str], num_launcher_replicas: Optional[int], num_workers: Optional[int], run_policy: Optional[RunPolicy], slots: int, verbose: Optional[bool], worker: Worker]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/error_handling.py + DOC106: Function `is_recoverable_worker_error`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Function `is_recoverable_worker_error`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC105: Function `is_recoverable_worker_error`: Argument names match, but type hints in these args do not match: failure +-------------------- +plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py + DOC001: Class `RunPolicy`: Potential formatting errors in docstring. Error message: Expected a colon in 'can remain active before it is terminated. Must be a positive integer. This setting applies only to pods.'. + DOC601: Class `RunPolicy`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `RunPolicy`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [active_deadline_seconds: Optional[int], backoff_limit: Optional[int], clean_pod_policy: CleanPodPolicy, ttl_seconds_after_finished: Optional[int]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `PyTorch`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `PyTorch`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [increase_shared_mem: bool, master: Master, num_workers: Optional[int], run_policy: Optional[RunPolicy], worker: Worker]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `Elastic`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Elastic`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [increase_shared_mem: bool, max_restarts: int, monitor_interval: int, nnodes: Union[int, str], nproc_per_node: int, rdzv_configs: Dict[str, Any], run_policy: Optional[RunPolicy], start_method: str]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `ElasticWorkerResult`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ElasticWorkerResult`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [om: Optional[OutputMetadata]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC101: Function `spawn_helper`: Docstring contains fewer arguments than in function signature. + DOC107: Function `spawn_helper`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints + DOC103: Function `spawn_helper`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [kwargs: ]. + DOC501: Function `spawn_helper` has "raise" statements, but the docstring does not have a "Raises" section + DOC503: Function `spawn_helper` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: []. Raised exceptions in the body: ['Exception']. + DOC101: Method `PytorchElasticFunctionTask._execute`: Docstring contains fewer arguments than in function signature. + DOC106: Method `PytorchElasticFunctionTask._execute`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC103: Method `PytorchElasticFunctionTask._execute`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: ]. + DOC203: Method `PytorchElasticFunctionTask._execute` return type(s) in docstring not consistent with the return annotation. Return annotation types: ['Any']; docstring return section types: [''] + DOC503: Method `PytorchElasticFunctionTask._execute` exceptions in the "Raises" section in the docstring do not match those in the function body. Raised exceptions in the docstring: ['FlyteRecoverableException', 'IgnoreOutputs', 'RuntimeError']. Raised exceptions in the body: ['FlyteRecoverableException', 'FlyteUserRuntimeException', 'IgnoreOutputs', 'ImportError', 'ValueError']. +-------------------- +plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py + DOC601: Class `RunPolicy`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `RunPolicy`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [active_deadline_seconds: Optional[int], backoff_limit: Optional[int], clean_pod_policy: CleanPodPolicy, ttl_seconds_after_finished: Optional[int]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `TfJob`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `TfJob`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [chief: Chief, evaluator: Evaluator, num_chief_replicas: Optional[int], num_evaluator_replicas: Optional[int], num_ps_replicas: Optional[int], num_workers: Optional[int], ps: PS, run_policy: Optional[RunPolicy], worker: Worker]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-memray/flytekitplugins/memray/profiling.py + DOC301: Class `memray_profiling`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-neptune/flytekitplugins/neptune/tracking.py + DOC105: Function `neptune_init_run`: Argument names match, but type hints in these args do not match: secret + DOC201: Function `neptune_init_run` does not have a return section in docstring + DOC301: Class `_neptune_init_run_class`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-omegaconf/flytekitplugins/omegaconf/dictconfig_transformer.py + DOC301: Class `DictConfigTransformer`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-omegaconf/flytekitplugins/omegaconf/listconfig_transformer.py + DOC301: Class `ListConfigTransformer`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py + DOC601: Class `PyTorch2ONNXConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `PyTorch2ONNXConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [args: Union[Tuple, torch.Tensor], custom_opsets: Dict[str, int], do_constant_folding: bool, dynamic_axes: Union[Dict[str, Dict[int, str]], Dict[str, List[int]]], export_modules_as_functions: Union[bool, set[Type]], export_params: bool, input_names: List[str], keep_initializers_as_inputs: Optional[bool], operator_export_type: Optional[torch.onnx.OperatorExportTypes], opset_version: int, output_names: List[str], training: torch.onnx.TrainingMode, verbose: bool]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-onnx-scikitlearn/flytekitplugins/onnxscikitlearn/schema.py + DOC601: Class `ScikitLearn2ONNXConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ScikitLearn2ONNXConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [black_op: Optional[Set[str]], custom_conversion_functions: Dict[Callable[..., Any], Callable[..., None]], custom_parsers: Dict[Callable[..., Any], Callable[..., None]], custom_shape_calculators: Dict[Callable[..., Any], Callable[..., None]], doc_string: str, final_types: Optional[List[Tuple[str, Type]]], initial_types: List[Tuple[str, Type]], intermediate: bool, name: Optional[str], naming: Optional[Union[str, Callable[..., Any]]], options: Dict[Any, Any], target_opset: Optional[int], verbose: int, white_op: Optional[Set[str]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-onnx-tensorflow/flytekitplugins/onnxtensorflow/schema.py + DOC601: Class `TensorFlow2ONNXConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `TensorFlow2ONNXConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [custom_op_handlers: Optional[Dict[Any, Tuple]], custom_ops: Optional[Dict[str, Any]], custom_rewriter: Optional[List[Any]], extra_opset: Optional[List[int]], input_signature: Union[tf.TensorSpec, np.ndarray], inputs_as_nchw: Optional[List[str]], large_model: bool, opset: Optional[int], shape_override: Optional[Dict[str, List[Any]]], target: Optional[List[Any]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-openai/flytekitplugins/openai/chatgpt/task.py + DOC301: Class `ChatGPTTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-optuna/flytekitplugins/optuna/optimizer.py + DOC109: Method `Optimizer.__call__`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list + DOC105: Method `Optimizer.__call__`: Argument names match, but type hints in these args do not match: **inputs +-------------------- +plugins/flytekit-snowflake/flytekitplugins/snowflake/task.py + DOC301: Class `SnowflakeTask`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-spark/flytekitplugins/spark/models.py + DOC301: Class `SparkJob`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- +plugins/flytekit-spark/flytekitplugins/spark/task.py + DOC601: Class `Spark`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `Spark`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [applications_path: Optional[str], executor_path: Optional[str], hadoop_conf: Optional[Dict[str, str]], spark_conf: Optional[Dict[str, str]]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC601: Class `DatabricksV2`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `DatabricksV2`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [databricks_conf: Optional[Dict[str, Union[str, dict]]], databricks_instance: Optional[str]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/task.py + DOC601: Class `SQLAlchemyConfig`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `SQLAlchemyConfig`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [connect_args: typing.Optional[typing.Dict[str, typing.Any]], secret_connect_args: typing.Optional[typing.Dict[str, Secret]], uri: str]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) +-------------------- +plugins/flytekit-wandb/flytekitplugins/wandb/tracking.py + DOC301: Class `wandb_init`: __init__() should not have a docstring; please combine it with the docstring of the class +-------------------- diff --git a/pyproject.toml b/pyproject.toml index 765d66e5e8..3452e8954b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,5 +146,5 @@ extend-exclude = ["tests/", "**/tests/**"] ] [tool.codespell] -ignore-words-list = "ot,te,raison,fo,lits" +ignore-words-list = "ot,te,raison,fo,lits,assertIn" skip = "./docs/build,./.git,*.txt" From d6405ccfd33fdd179d9fc38aa6155bf56445baac Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Fri, 24 Jan 2025 20:14:16 -0500 Subject: [PATCH 23/27] Also run unit tests on arm64 (#3085) Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index d6f98fb771..303362336f 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -39,7 +39,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: + - ubuntu-24.04-arm + - ubuntu-latest + - windows-latest + - macos-latest python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} steps: - uses: actions/checkout@v4 @@ -78,7 +82,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: + - ubuntu-24.04-arm + - ubuntu-latest + - windows-latest + - macos-latest python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} steps: - uses: actions/checkout@v4 From b61436b535d16ed87b93de4c217fb4ed1071514a Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Fri, 24 Jan 2025 20:46:28 -0500 Subject: [PATCH 24/27] Regenerate baseline (#3090) Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- pydoclint-errors-baseline.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoclint-errors-baseline.txt b/pydoclint-errors-baseline.txt index 1ac90bc20c..e3fd80d99d 100644 --- a/pydoclint-errors-baseline.txt +++ b/pydoclint-errors-baseline.txt @@ -176,7 +176,7 @@ flytekit/extras/tensorflow/record.py -------------------- flytekit/image_spec/image_spec.py DOC601: Class `ImageSpec`: Class docstring contains fewer class attributes than actual class attributes. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) - DOC603: Class `ImageSpec`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [apt_packages: Optional[List[str]], base_image: Optional[Union[str, 'ImageSpec']], builder: Optional[str], commands: Optional[List[str]], conda_channels: Optional[List[str]], conda_packages: Optional[List[str]], copy: Optional[List[str]], cuda: Optional[str], cudnn: Optional[str], entrypoint: Optional[List[str]], env: Optional[typing.Dict[str, str]], name: str, packages: Optional[List[str]], pip_extra_index_url: Optional[List[str]], pip_index: Optional[str], platform: str, python_exec: Optional[str], python_version: str, registry: Optional[str], registry_config: Optional[str], requirements: Optional[str], source_copy_mode: Optional[CopyFileDetection], source_root: Optional[str], tag_format: Optional[str]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) + DOC603: Class `ImageSpec`: Class docstring attributes are different from actual class attributes. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Attributes in the class definition but not in the docstring: [apt_packages: Optional[List[str]], base_image: Optional[Union[str, 'ImageSpec']], builder: Optional[str], commands: Optional[List[str]], conda_channels: Optional[List[str]], conda_packages: Optional[List[str]], copy: Optional[List[str]], cuda: Optional[str], cudnn: Optional[str], entrypoint: Optional[List[str]], env: Optional[typing.Dict[str, str]], name: str, packages: Optional[List[str]], pip_extra_args: Optional[str], pip_extra_index_url: Optional[List[str]], pip_index: Optional[str], platform: str, python_exec: Optional[str], python_version: str, registry: Optional[str], registry_config: Optional[str], requirements: Optional[str], source_copy_mode: Optional[CopyFileDetection], source_root: Optional[str], tag_format: Optional[str]]. (Please read https://jsh9.github.io/pydoclint/checking_class_attributes.html on how to correctly document class attributes.) DOC109: Method `ImageSpecBuilder.build_image`: The option `--arg-type-hints-in-docstring` is `True` but there are no type hints in the docstring arg list DOC110: Method `ImageSpecBuilder.build_image`: The option `--arg-type-hints-in-docstring` is `True` but not all args in the docstring arg list have type hints DOC105: Method `ImageSpecBuilder.build_image`: Argument names match, but type hints in these args do not match: image_spec From f0ba47fd4fe353b9a5b20982d6b5408231cabfb1 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 24 Jan 2025 22:15:22 -0800 Subject: [PATCH 25/27] silence union warning (#3091) Signed-off-by: Yee Hing Tong --- flytekit/core/type_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 288b48077a..7019127ffa 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -1908,7 +1908,7 @@ async def async_to_literal( res_type = _add_tag_to_type(trans.get_literal_type(t), trans.name) found_res = True except Exception as e: - logger.warning( + logger.debug( f"UnionTransformer failed attempt to convert from {python_val} to {t} error: {e}", ) continue From 4208a641debb0334c49c9331bcc4d98ed5c45d12 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 27 Jan 2025 09:25:42 -0800 Subject: [PATCH 26/27] Add request and limit to ray config (#3087) * Add request and limit to ray config Signed-off-by: Kevin Su * lint Signed-off-by: Kevin Su * nit Signed-off-by: Kevin Su * nit Signed-off-by: Kevin Su * update flytekit version Signed-off-by: Kevin Su * nit Signed-off-by: Kevin Su --------- Signed-off-by: Kevin Su Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- flytekit/core/resources.py | 12 ++-- flytekit/models/task.py | 20 ++++++ .../flytekit-ray/flytekitplugins/ray/task.py | 65 +++++++++++++++---- plugins/flytekit-ray/setup.py | 2 +- plugins/flytekit-ray/tests/test_ray.py | 26 ++++++-- tests/flytekit/unit/core/test_resources.py | 16 ++--- 6 files changed, 109 insertions(+), 32 deletions(-) diff --git a/flytekit/core/resources.py b/flytekit/core/resources.py index 9a334f98f6..f64b7d23dc 100644 --- a/flytekit/core/resources.py +++ b/flytekit/core/resources.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, fields -from typing import Any, List, Optional, Union +from typing import List, Optional, Union from kubernetes.client import V1Container, V1PodSpec, V1ResourceRequirements from mashumaro.mixins.json import DataClassJSONMixin @@ -103,11 +103,11 @@ def convert_resources_to_resource_model( def pod_spec_from_resources( - k8s_pod_name: str, + primary_container_name: Optional[str] = None, requests: Optional[Resources] = None, limits: Optional[Resources] = None, k8s_gpu_resource_key: str = "nvidia.com/gpu", -) -> dict[str, Any]: +) -> V1PodSpec: def _construct_k8s_pods_resources(resources: Optional[Resources], k8s_gpu_resource_key: str): if resources is None: return None @@ -133,10 +133,10 @@ def _construct_k8s_pods_resources(resources: Optional[Resources], k8s_gpu_resour requests = requests or limits limits = limits or requests - k8s_pod = V1PodSpec( + pod_spec = V1PodSpec( containers=[ V1Container( - name=k8s_pod_name, + name=primary_container_name, resources=V1ResourceRequirements( requests=requests, limits=limits, @@ -145,4 +145,4 @@ def _construct_k8s_pods_resources(resources: Optional[Resources], k8s_gpu_resour ] ) - return k8s_pod.to_dict() + return pod_spec diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 960555fd9b..9693390458 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -8,6 +8,7 @@ from flyteidl.core import tasks_pb2 as _core_task from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct +from kubernetes.client import ApiClient from flytekit.models import common as _common from flytekit.models import interface as _interface @@ -16,6 +17,9 @@ from flytekit.models.core import identifier as _identifier from flytekit.models.documentation import Documentation +if typing.TYPE_CHECKING: + from flytekit import PodTemplate + class Resources(_common.FlyteIdlEntity): class ResourceName(object): @@ -1042,6 +1046,22 @@ def from_flyte_idl(cls, pb2_object: _core_task.K8sPod): else None, ) + def to_pod_template(self) -> "PodTemplate": + from flytekit import PodTemplate + + return PodTemplate( + labels=self.metadata.labels, + annotations=self.metadata.annotations, + pod_spec=self.pod_spec, + ) + + @classmethod + def from_pod_template(cls, pod_template: "PodTemplate") -> "K8sPod": + return cls( + metadata=K8sObjectMetadata(labels=pod_template.labels, annotations=pod_template.annotations), + pod_spec=ApiClient().sanitize_for_serialization(pod_template.pod_spec), + ) + class Sql(_common.FlyteIdlEntity): class Dialect(object): diff --git a/plugins/flytekit-ray/flytekitplugins/ray/task.py b/plugins/flytekit-ray/flytekitplugins/ray/task.py index 98a653a990..c87c86276d 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/task.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/task.py @@ -14,20 +14,30 @@ ) from google.protobuf.json_format import MessageToDict -from flytekit import lazy_module +from flytekit import PodTemplate, Resources, lazy_module from flytekit.configuration import SerializationSettings from flytekit.core.context_manager import ExecutionParameters, FlyteContextManager from flytekit.core.python_function_task import PythonFunctionTask +from flytekit.core.resources import pod_spec_from_resources from flytekit.extend import TaskPlugins from flytekit.models.task import K8sPod ray = lazy_module("ray") +_RAY_HEAD_CONTAINER_NAME = "ray-head" +_RAY_WORKER_CONTAINER_NAME = "ray-worker" @dataclass class HeadNodeConfig: ray_start_params: typing.Optional[typing.Dict[str, str]] = None - k8s_pod: typing.Optional[K8sPod] = None + pod_template: typing.Optional[PodTemplate] = None + requests: Optional[Resources] = None + limits: Optional[Resources] = None + + def __post_init__(self): + if self.pod_template: + if self.requests and self.limits: + raise ValueError("Cannot specify both pod_template and requests/limits") @dataclass @@ -37,7 +47,14 @@ class WorkerNodeConfig: min_replicas: typing.Optional[int] = None max_replicas: typing.Optional[int] = None ray_start_params: typing.Optional[typing.Dict[str, str]] = None - k8s_pod: typing.Optional[K8sPod] = None + pod_template: typing.Optional[PodTemplate] = None + requests: Optional[Resources] = None + limits: Optional[Resources] = None + + def __post_init__(self): + if self.pod_template: + if self.requests and self.limits: + raise ValueError("Cannot specify both pod_template and requests/limits") @dataclass @@ -83,25 +100,49 @@ def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters: def get_custom(self, settings: SerializationSettings) -> Optional[Dict[str, Any]]: cfg = self._task_config - # Deprecated: runtime_env is removed KubeRay >= 1.1.0. It is replaced by runtime_env_yaml runtime_env = base64.b64encode(json.dumps(cfg.runtime_env).encode()).decode() if cfg.runtime_env else None - runtime_env_yaml = yaml.dump(cfg.runtime_env) if cfg.runtime_env else None + if cfg.head_node_config.requests or cfg.head_node_config.limits: + head_pod_template = PodTemplate( + pod_spec=pod_spec_from_resources( + primary_container_name=_RAY_HEAD_CONTAINER_NAME, + requests=cfg.head_node_config.requests, + limits=cfg.head_node_config.limits, + ) + ) + else: + head_pod_template = cfg.head_node_config.pod_template + + worker_group_spec: typing.List[WorkerGroupSpec] = [] + for c in cfg.worker_node_config: + if c.requests or c.limits: + worker_pod_template = PodTemplate( + pod_spec=pod_spec_from_resources( + primary_container_name=_RAY_WORKER_CONTAINER_NAME, + requests=c.requests, + limits=c.limits, + ) + ) + else: + worker_pod_template = c.pod_template + k8s_pod = K8sPod.from_pod_template(worker_pod_template) if worker_pod_template else None + worker_group_spec.append( + WorkerGroupSpec(c.group_name, c.replicas, c.min_replicas, c.max_replicas, c.ray_start_params, k8s_pod) + ) + ray_job = RayJob( ray_cluster=RayCluster( head_group_spec=( - HeadGroupSpec(cfg.head_node_config.ray_start_params, cfg.head_node_config.k8s_pod) + HeadGroupSpec( + cfg.head_node_config.ray_start_params, + K8sPod.from_pod_template(head_pod_template) if head_pod_template else None, + ) if cfg.head_node_config else None ), - worker_group_spec=[ - WorkerGroupSpec( - c.group_name, c.replicas, c.min_replicas, c.max_replicas, c.ray_start_params, c.k8s_pod - ) - for c in cfg.worker_node_config - ], + worker_group_spec=worker_group_spec, enable_autoscaling=(cfg.enable_autoscaling if cfg.enable_autoscaling else False), ), runtime_env=runtime_env, diff --git a/plugins/flytekit-ray/setup.py b/plugins/flytekit-ray/setup.py index 18b95498ee..2237be2030 100644 --- a/plugins/flytekit-ray/setup.py +++ b/plugins/flytekit-ray/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["ray[default]", "flytekit>=1.3.0b2,<2.0.0", "flyteidl>=1.13.6"] +plugin_requires = ["ray[default]", "flytekit>1.14.5", "flyteidl>=1.13.6"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-ray/tests/test_ray.py b/plugins/flytekit-ray/tests/test_ray.py index c943067013..8fd8d432a9 100644 --- a/plugins/flytekit-ray/tests/test_ray.py +++ b/plugins/flytekit-ray/tests/test_ray.py @@ -3,6 +3,8 @@ import ray import yaml + +from flytekit.core.resources import pod_spec_from_resources from flytekitplugins.ray import HeadNodeConfig from flytekitplugins.ray.models import ( HeadGroupSpec, @@ -13,10 +15,17 @@ from flytekitplugins.ray.task import RayJobConfig, WorkerNodeConfig from google.protobuf.json_format import MessageToDict -from flytekit import PythonFunctionTask, task +from flytekit import PythonFunctionTask, task, PodTemplate, Resources from flytekit.configuration import Image, ImageConfig, SerializationSettings from flytekit.models.task import K8sPod + +pod_template=PodTemplate( + primary_container_name="primary", + labels={"lKeyA": "lValA"}, + annotations={"aKeyA": "aValA"}, + ) + config = RayJobConfig( worker_node_config=[ WorkerNodeConfig( @@ -24,10 +33,10 @@ replicas=3, min_replicas=0, max_replicas=10, - k8s_pod=K8sPod(pod_spec={"str": "worker", "int": 1}), + pod_template=pod_template, ) ], - head_node_config=HeadNodeConfig(k8s_pod=K8sPod(pod_spec={"str": "head", "int": 2})), + head_node_config=HeadNodeConfig(requests=Resources(cpu="1", mem="1Gi"), limits=Resources(cpu="2", mem="2Gi")), runtime_env={"pip": ["numpy"]}, enable_autoscaling=True, shutdown_after_job_finishes=True, @@ -55,6 +64,13 @@ def t1(a: int) -> str: image_config=ImageConfig(default_image=default_img, images=[default_img]), env={}, ) + head_pod_template = PodTemplate( + pod_spec=pod_spec_from_resources( + primary_container_name="ray-head", + requests=Resources(cpu="1", mem="1Gi"), + limits=Resources(cpu="2", mem="2Gi"), + ) + ) ray_job_pb = RayJob( ray_cluster=RayCluster( @@ -64,10 +80,10 @@ def t1(a: int) -> str: replicas=3, min_replicas=0, max_replicas=10, - k8s_pod=K8sPod(pod_spec={"str": "worker", "int": 1}), + k8s_pod=K8sPod.from_pod_template(pod_template), ) ], - head_group_spec=HeadGroupSpec(k8s_pod=K8sPod(pod_spec={"str": "head", "int": 2})), + head_group_spec=HeadGroupSpec(k8s_pod=K8sPod.from_pod_template(head_pod_template)), enable_autoscaling=True, ), runtime_env=base64.b64encode(json.dumps({"pip": ["numpy"]}).encode()).decode(), diff --git a/tests/flytekit/unit/core/test_resources.py b/tests/flytekit/unit/core/test_resources.py index 1c09a111e3..115605b055 100644 --- a/tests/flytekit/unit/core/test_resources.py +++ b/tests/flytekit/unit/core/test_resources.py @@ -110,12 +110,12 @@ def test_resources_round_trip(): def test_pod_spec_from_resources_requests_limits_set(): requests = Resources(cpu="1", mem="1Gi", gpu="1", ephemeral_storage="1Gi") limits = Resources(cpu="4", mem="2Gi", gpu="1", ephemeral_storage="1Gi") - k8s_pod_name = "foo" + primary_container_name = "foo" expected_pod_spec = V1PodSpec( containers=[ V1Container( - name=k8s_pod_name, + name=primary_container_name, resources=V1ResourceRequirements( requests={ "cpu": "1", @@ -133,19 +133,19 @@ def test_pod_spec_from_resources_requests_limits_set(): ) ] ) - pod_spec = pod_spec_from_resources(k8s_pod_name=k8s_pod_name, requests=requests, limits=limits) - assert expected_pod_spec == V1PodSpec(**pod_spec) + pod_spec = pod_spec_from_resources(primary_container_name=primary_container_name, requests=requests, limits=limits) + assert expected_pod_spec == pod_spec def test_pod_spec_from_resources_requests_set(): requests = Resources(cpu="1", mem="1Gi") limits = None - k8s_pod_name = "foo" + primary_container_name = "foo" expected_pod_spec = V1PodSpec( containers=[ V1Container( - name=k8s_pod_name, + name=primary_container_name, resources=V1ResourceRequirements( requests={"cpu": "1", "memory": "1Gi"}, limits={"cpu": "1", "memory": "1Gi"}, @@ -153,5 +153,5 @@ def test_pod_spec_from_resources_requests_set(): ) ] ) - pod_spec = pod_spec_from_resources(k8s_pod_name=k8s_pod_name, requests=requests, limits=limits) - assert expected_pod_spec == V1PodSpec(**pod_spec) + pod_spec = pod_spec_from_resources(primary_container_name=primary_container_name, requests=requests, limits=limits) + assert expected_pod_spec == pod_spec From 88ac611e52309dee6322edea57c764582293e513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E5=AE=B6=E7=91=8B?= <36886416+JiangJiaWei1103@users.noreply.github.com> Date: Sat, 1 Feb 2025 01:58:36 +0800 Subject: [PATCH 27/27] [BUG] Fix StructuredDataset empty-str `file_format` in dc attr access (#3027) * fix: Retain user-specified file format info Signed-off-by: JiaWei Jiang * fix: Set sdt format based on user-specified file_format Signed-off-by: JiaWei Jiang * Remove redundant modification Signed-off-by: JiaWei Jiang * test: Test file_format attribute alignment in dc.sd Signed-off-by: JiaWei Jiang * Merge master and support pqt file upload Signed-off-by: JiaWei Jiang * Remove redundant condition to always copy file_format over Signed-off-by: JiangJiaWei1103 * Prioritize file_format in type hint over the user-specified one Signed-off-by: JiangJiaWei1103 --------- Signed-off-by: JiaWei Jiang Signed-off-by: JiangJiaWei1103 Co-authored-by: Future-Outlier --- .../types/structured/structured_dataset.py | 21 ++++++ .../integration/remote/test_remote.py | 46 +++++++++++++ .../remote/workflows/basic/sd_attr.py | 68 +++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 tests/flytekit/integration/remote/workflows/basic/sd_attr.py diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index 4812ce0856..7dd9532382 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -739,10 +739,31 @@ async def async_to_literal( # return StructuredDataset(uri=uri) if python_val.dataframe is None: uri = python_val.uri + file_format = python_val.file_format + + # Check the user-specified uri if not uri: raise ValueError(f"If dataframe is not specified, then the uri should be specified. {python_val}") if not ctx.file_access.is_remote(uri): uri = await ctx.file_access.async_put_raw_data(uri) + + # Check the user-specified file_format + # When users specify file_format for a StructuredDataset, the file_format should be retained conditionally. + # For details, please refer to https://github.com/flyteorg/flyte/issues/6096. + # Following illustrates why we can't always copy the user-specified file_format over: + # + # @task + # def modify_format(sd: Annotated[StructuredDataset, {}, "task-format"]) -> StructuredDataset: + # return sd + # + # sd = StructuredDataset(uri="s3://my-s3-bucket/df.parquet", file_format="user-format") + # sd2 = modify_format(sd=sd) + # + # In this case, we expect sd2.file_format to be task-format (as shown in Annotated), not user-format. + # If we directly copy the user-specified file_format over, the type hint information will be missing. + if sdt.format == GENERIC_FORMAT and file_format != GENERIC_FORMAT: + sdt.format = file_format + sd_model = literals.StructuredDataset( uri=uri, metadata=StructuredDatasetMetadata(structured_dataset_type=sdt), diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 39ff09f169..ea0a788c49 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -16,6 +16,7 @@ import uuid import pytest from unittest import mock +from dataclasses import dataclass from flytekit import LaunchPlan, kwtypes, WorkflowExecutionPhase from flytekit.configuration import Config, ImageConfig, SerializationSettings @@ -27,6 +28,7 @@ from flytekit.remote.remote import FlyteRemote from flyteidl.service import dataproxy_pb2 as _data_proxy_pb2 from flytekit.types.schema import FlyteSchema +from flytekit.types.structured import StructuredDataset from flytekit.clients.friendly import SynchronousFlyteClient as _SynchronousFlyteClient from flytekit.configuration import PlatformConfig @@ -877,6 +879,50 @@ def test_attr_access_sd(): bucket, key = url.netloc, url.path.lstrip("/") file_transfer.delete_file(bucket=bucket, key=key) + +def test_sd_attr(): + """Test correctness of StructuredDataset attributes. + + This test considers only the following condition: + 1. Check StructuredDataset (wrapped in a dataclass) file_format attribute + + We'll make sure uri aligns with the user-specified one in the future. + """ + from workflows.basic.sd_attr import wf + + @dataclass + class DC: + sd: StructuredDataset + + FILE_FORMAT = "parquet" + + # Upload a file to minio s3 bucket + file_transfer = SimpleFileTransfer() + remote_file_path = file_transfer.upload_file(file_type=FILE_FORMAT) + + # Create a dataclass as the workflow input because `pyflyte run` + # can't properly handle input arg `dc` as a json str so far + dc = DC(sd=StructuredDataset(uri=remote_file_path, file_format=FILE_FORMAT)) + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN, interactive_mode_enabled=True) + wf_exec = remote.execute( + wf, + inputs={"dc": dc, "file_format": FILE_FORMAT}, + wait=True, + version=VERSION, + image_config=ImageConfig.from_images(IMAGE), + ) + assert wf_exec.closure.phase == WorkflowExecutionPhase.SUCCEEDED, f"Execution failed with phase: {wf_exec.closure.phase}" + assert wf_exec.outputs["o0"].file_format == FILE_FORMAT, ( + f"Workflow output StructuredDataset file_format should align with the user-specified file_format: {FILE_FORMAT}." + ) + + # Delete the remote file to free the space + url = urlparse(remote_file_path) + bucket, key = url.netloc, url.path.lstrip("/") + file_transfer.delete_file(bucket=bucket, key=key) + + def test_signal_approve_reject(register): from flytekit.models.types import LiteralType, SimpleType from time import sleep diff --git a/tests/flytekit/integration/remote/workflows/basic/sd_attr.py b/tests/flytekit/integration/remote/workflows/basic/sd_attr.py new file mode 100644 index 0000000000..b357d52fa4 --- /dev/null +++ b/tests/flytekit/integration/remote/workflows/basic/sd_attr.py @@ -0,0 +1,68 @@ +from dataclasses import dataclass + +import pandas as pd +from flytekit import task, workflow +from flytekit.types.structured import StructuredDataset + + +@dataclass +class DC: + sd: StructuredDataset + + +@task +def create_dc(uri: str, file_format: str) -> DC: + """Create a dataclass with a StructuredDataset attribute. + + Args: + uri: File URI. + file_format: File format, e.g., parquet, csv. + + Returns: + dc: A dataclass with a StructuredDataset attribute. + """ + dc = DC(sd=StructuredDataset(uri=uri, file_format=file_format)) + + return dc + + +@task +def check_file_format(sd: StructuredDataset, true_file_format: str) -> StructuredDataset: + """Check StructuredDataset file_format attribute. + + StruturedDataset file_format should align with what users specify. + + Args: + sd: Python native StructuredDataset. + true_file_format: User-specified file_format. + """ + assert sd.file_format == true_file_format, ( + f"StructuredDataset file_format should align with the user-specified file_format: {true_file_format}." + ) + assert sd._literal_sd.metadata.structured_dataset_type.format == true_file_format, ( + f"StructuredDatasetType format should align with the user-specified file_format: {true_file_format}." + ) + print(f">>> SD <<<\n{sd}") + print(f">>> Literal SD <<<\n{sd._literal_sd}") + print(f">>> SDT <<<\n{sd._literal_sd.metadata.structured_dataset_type}") + print(f">>> DF <<<\n{sd.open(pd.DataFrame).all()}") + + return sd + + +@workflow +def wf(dc: DC, file_format: str) -> StructuredDataset: + # Fail to use dc.sd.file_format as the input + sd = check_file_format(sd=dc.sd, true_file_format=file_format) + + return sd + + +if __name__ == "__main__": + # Define inputs + uri = "tests/flytekit/integration/remote/workflows/basic/data/df.parquet" + file_format = "parquet" + + dc = create_dc(uri=uri, file_format=file_format) + sd = wf(dc=dc, file_format=file_format) + print(sd.file_format)