update

src/litserve/loops/continuous_batching_loop.py

@@ -157,7 +157,6 @@ async def run_in_background(
         try:
             pending_requests = []
             while True:
-                await asyncio.sleep(0.001)
                 pending_requests = await self.prefill(
                     pending_requests,
                     lit_api,
@@ -167,6 +166,7 @@ async def run_in_background(
                     batch_timeout,
                     response_queues,
                 )
+                await asyncio.sleep(0)
         except Exception as e:
             logger.exception("An error occurred in run_in_background: %s", e)
         finally:
@@ -194,7 +194,7 @@ async def run(
         """Main loop that processes batches of requests."""
         try:
             prev_outputs = None
-            while await lit_api.has_active_requests():
+            while lit_api.has_active_requests():
                 # Process one step for all active sequences
                 responses = await self.step(prev_outputs, lit_api, lit_spec)
                 if len(responses) == 0:

src/litserve/loops/loops.py

@@ -476,7 +476,7 @@ async def _wrapper():
                             workers_setup_status,
                             callback_runner,
                         )
-                        await asyncio.sleep(0.00001)
+                        await asyncio.sleep(0)
                     except Exception as e:
                         logger.exception("An error occurred in the loop: %s", e)
                         # Optionally, break the loop or handle the error as needed