Skip to content

Commit 1fa71ab

Browse files
committed
update validation
1 parent b47bc10 commit 1fa71ab

File tree

1 file changed

+12
-5
lines changed

1 file changed

+12
-5
lines changed

src/litserve/loops.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -725,11 +725,18 @@ def pre_setup(self, lit_api: LitAPI, spec: Optional[LitSpec]):
725725
"Continuous batching loop requires streaming to be enabled. Please set LitServe(..., stream=True)"
726726
)
727727

728-
if lit_api.max_batch_size <= 1:
729-
raise ValueError(
730-
"Continuous batching loop requires max_batch_size to be greater than 1. "
731-
"Please set LitServe(..., max_batch_size=2)"
732-
)
728+
if not hasattr(lit_api, "step") and not hasattr(lit_api, "predict"):
729+
raise ValueError("""Using the default step method with Continuous batching loop requires the lit_api to
730+
have a `predict` method which accepts decoded request inputs and a list of generated_sequence.
731+
Please implement the has_finished method in the lit_api.
732+
733+
class ExampleAPI(LitAPI):
734+
...
735+
def predict(self, inputs, generated_sequence):
736+
# implement predict logic
737+
# return list of new tokens
738+
...
739+
""")
733740

734741
if not hasattr(lit_api, "step") and not hasattr(lit_api, "has_finished"):
735742
raise ValueError("""Using the default step method with Continuous batching loop

0 commit comments

Comments
 (0)