From b5ff91a12f2c49352e11fed7e582fbb843206836 Mon Sep 17 00:00:00 2001 From: Aniket Maurya Date: Thu, 19 Dec 2024 19:11:55 +0000 Subject: [PATCH] update --- src/litserve/loops/continuous_batching_loop.py | 4 ++-- src/litserve/loops/loops.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/litserve/loops/continuous_batching_loop.py b/src/litserve/loops/continuous_batching_loop.py index 4cd207af..77d364ef 100644 --- a/src/litserve/loops/continuous_batching_loop.py +++ b/src/litserve/loops/continuous_batching_loop.py @@ -157,7 +157,6 @@ async def run_in_background( try: pending_requests = [] while True: - await asyncio.sleep(0.001) pending_requests = await self.prefill( pending_requests, lit_api, @@ -167,6 +166,7 @@ async def run_in_background( batch_timeout, response_queues, ) + await asyncio.sleep(0) except Exception as e: logger.exception("An error occurred in run_in_background: %s", e) finally: @@ -194,7 +194,7 @@ async def run( """Main loop that processes batches of requests.""" try: prev_outputs = None - while await lit_api.has_active_requests(): + while lit_api.has_active_requests(): # Process one step for all active sequences responses = await self.step(prev_outputs, lit_api, lit_spec) if len(responses) == 0: diff --git a/src/litserve/loops/loops.py b/src/litserve/loops/loops.py index f67dc76e..911fd40c 100644 --- a/src/litserve/loops/loops.py +++ b/src/litserve/loops/loops.py @@ -476,7 +476,7 @@ async def _wrapper(): workers_setup_status, callback_runner, ) - await asyncio.sleep(0.00001) + await asyncio.sleep(0) except Exception as e: logger.exception("An error occurred in the loop: %s", e) # Optionally, break the loop or handle the error as needed