an exception occurred in remote runner [429] in production mode

[litter-rabbit]

when I use bentoml serve service.py:svc start server, then use locust to test it ,it work fine, but when i use bentoml serve service.py:svc --production then i use locust to test it,that will get many error like these.

Traceback (most recent call last):
 File "C:\Users\CDROG\anicaptcha-bentoml\src\anticaptcha_bentoml\hcaptcha\service.py", line 124, in invoke
   outputs = runner.run.run(imgs_array)
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\site-packages\bentoml\_internal\runner\runner.py", line 40, in run
   return self.runner._runner_handle.run_method(  # type: ignore
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\site-packages\bentoml\_internal\runner\runner_handle\remote.py", line 210, in run_method
   return anyio.from_thread.run(  # type: ignore (pyright cannot infer the return type)
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\site-packages\anyio\from_thread.py", line 49, in run
   return asynclib.run_async_from_thread(func, *args)
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\site-packages\anyio\_backends\_asyncio.py", line 970, in run_async_from_thread
   return f.result()
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\concurrent\futures\_base.py", line 446, in result
   return self.__get_result()
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\concurrent\futures\_base.py", line 391, in __get_result
   raise self._exception
 File "C:\Users\CDROG\ml\ml-muggle-deploy\env\lib\site-packages\bentoml\_internal\runner\runner_handle\remote.py", line 164, in async_run_method
   raise RemoteException(
bentoml.exceptions.RemoteException: An exception occurred in remote runner captcharunabel: [429]

and spend time get very long. i try to set max_batch_size, or max_latency_ms to 1000,1000 , but it's can't work .

here is my runner

class OnnxRunableBase(bentoml.Runnable,ABC):
    SUPPORTED_RESOURCES = ("nvidia.com/gpu", "cpu")
    SUPPORTS_CPU_MULTI_THREADING = True

    def __init__(self,model_name=None):
        model_path = os.path.join(base_dir,model_name)
        self.sess_options = onnxruntime.SessionOptions()
        self.session = onnxruntime.InferenceSession(model_path, self.sess_options,providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
        self.outputs_names = [_.name for _ in self.session.get_outputs()]
        self.inputs_names = [_.name for _ in self.session.get_inputs()]

    def _onnx_run(self,img):
        return self.session.run(self.outputs_names, {self.inputs_names[0]: img})[0]

    @abstractmethod
    def onnx_run_before(self,img) -> np.array:
        pass

    @abstractmethod
    def onnx_run_after(self,output) -> np.array:
        pass

    @bentoml.Runnable.method(batchable=True, batch_dim=0)
    def run(self,input):
        img_arr = self.onnx_run_before(input)
        output = self._onnx_run(img_arr)
        return self.onnx_run_after(output)

    @property
    def model_name(self):
        return self.model_name