Skip to content

Commit d07c076

Browse files
authored
add validation for stream=False and yield usage
1 parent e9b1ee1 commit d07c076

File tree

1 file changed

+23
-0
lines changed

1 file changed

+23
-0
lines changed

src/litserve/loops/base.py

+23
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,29 @@ def pre_setup(self, lit_api: LitAPI, spec: Optional[LitSpec]):
264264
return
265265

266266
original = lit_api.unbatch.__code__ is LitAPI.unbatch.__code__
267+
if not lit_api.stream and any([
268+
inspect.isgeneratorfunction(lit_api.predict),
269+
inspect.isgeneratorfunction(lit_api.encode_response),
270+
(original or inspect.isgeneratorfunction(lit_api.unbatch)),
271+
]):
272+
raise ValueError(
273+
"""When `stream=False`, `lit_api.predict`, `lit_api.encode_response` and `lit_api.unbatch` must not be
274+
generator functions.
275+
276+
Correct example:
277+
278+
def predict(self, inputs):
279+
...
280+
return {"output": output}
281+
282+
Incorrect example:
283+
284+
def predict(self, inputs):
285+
...
286+
for i in range(max_token_length):
287+
yield prediction
288+
"""
289+
)
267290
if (
268291
lit_api.stream
269292
and lit_api.max_batch_size > 1

0 commit comments

Comments
 (0)