Skip to content

Instantly share code, notes, and snippets.

@crypdick

crypdick/flaky_ray_data_from_workspace.log

Created June 18, 2025 00:14
Show Gist options
  • Save crypdick/7b3781b96687b964551668e1c174b97c to your computer and use it in GitHub Desktop.
Save crypdick/7b3781b96687b964551668e1c174b97c to your computer and use it in GitHub Desktop.
Download ZIP
Raw
flaky_ray_data_from_workspace.log
<deleted 50,000 lines of logs repeating the same thing>
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 317, in generate_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) output = await self._generate_async(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/ray/llm/_internal/batch/stages/vllm_engine_stage.py", line 399, in generate_async_v1
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) async for request_output in stream:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 306, in generate
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) out = q.get_nowait() or await q.get()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/output_processor.py", line 51, in get
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise output
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 357, in output_handler
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) outputs = await engine_core.get_output_async()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 716, in get_output_async
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise self._format_exception(outputs) from None
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause.
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) Process EngineCore_0:
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) Traceback (most recent call last):
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self.run()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/multiprocessing/process.py", line 108, in run
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self._target(*self._args, **self._kwargs)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 400, in run_engine_core
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) raise e
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 389, in run_engine_core
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) engine_core.run_busy_loop()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 411, in run_busy_loop
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self._process_input_queue()
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 424, in _process_input_queue
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self._handle_client_request(*req)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 448, in _handle_client_request
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self.add_request(request)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 179, in add_request
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self.structured_output_manager.grammar_init(req)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/structured_output/__init__.py", line 53, in grammar_init
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self.backend = XgrammarBackend(self.vllm_config)
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) File "/home/ray/anaconda3/lib/python3.11/site-packages/vllm/v1/structured_output/backend_xgrammar.py", line 91, in __init__
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) self.compiler = xgr.GrammarCompiler(
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) ^^^^^^^^^^^^^^^^^^^^
(MapWorker(MapBatches(vLLMEngineStageUDF)) pid=3858, ip=10.0.54.100) TypeError: GrammarCompiler.__init__() got an unexpected keyword argument 'cache_limit_bytes'
(_MapWorker pid=3856, ip=10.0.18.162) The product of batch_size (2) and max_concurrent_batches (8) is too small to saturate vLLM engine. This may lead to suboptimal throughput. Please increase max_concurrent_batches to at least 128.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment