Created
October 14, 2025 21:33
-
-
Save sozercan/191d6970dad683610497b114cede208c to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| oss-gpt120b-bench-94ldh Downloading setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (32 kB) | |
| oss-gpt120b-bench-94ldh Downloading soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl (1.3 MB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 115.8 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading textual-5.3.0-py3-none-any.whl (702 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 702.7/702.7 kB 71.6 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading tqdm-4.67.1-py3-none-any.whl (78 kB) | |
| oss-gpt120b-bench-94ldh Downloading transformers-4.57.1-py3-none-any.whl (12.0 MB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.0/12.0 MB 141.6 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.7/4.7 MB 146.3 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB) | |
| oss-gpt120b-bench-94ldh Downloading aiosignal-1.4.0-py3-none-any.whl (7.5 kB) | |
| oss-gpt120b-bench-94ldh Downloading annotated_types-0.7.0-py3-none-any.whl (13 kB) | |
| oss-gpt120b-bench-94ldh Downloading anyio-4.11.0-py3-none-any.whl (109 kB) | |
| oss-gpt120b-bench-94ldh Downloading attrs-25.4.0-py3-none-any.whl (67 kB) | |
| oss-gpt120b-bench-94ldh Downloading cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (219 kB) | |
| oss-gpt120b-bench-94ldh Downloading distro-1.9.0-py3-none-any.whl (20 kB) | |
| oss-gpt120b-bench-94ldh Downloading docstring_parser-0.17.0-py3-none-any.whl (36 kB) | |
| oss-gpt120b-bench-94ldh Downloading frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl (242 kB) | |
| oss-gpt120b-bench-94ldh Downloading httpx-0.28.1-py3-none-any.whl (73 kB) | |
| oss-gpt120b-bench-94ldh Downloading httpcore-1.0.9-py3-none-any.whl (78 kB) | |
| oss-gpt120b-bench-94ldh Downloading httpx_aiohttp-0.1.8-py3-none-any.whl (6.2 kB) | |
| oss-gpt120b-bench-94ldh Downloading huggingface_hub-0.35.3-py3-none-any.whl (564 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 564.3/564.3 kB 55.5 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading jiter-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (347 kB) | |
| oss-gpt120b-bench-94ldh Downloading markdown_it_py-4.0.0-py3-none-any.whl (87 kB) | |
| oss-gpt120b-bench-94ldh Downloading multidict-6.7.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (256 kB) | |
| oss-gpt120b-bench-94ldh Using cached packaging-25.0-py3-none-any.whl (66 kB) | |
| oss-gpt120b-bench-94ldh Downloading platformdirs-4.5.0-py3-none-any.whl (18 kB) | |
| oss-gpt120b-bench-94ldh Downloading propcache-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (221 kB) | |
| oss-gpt120b-bench-94ldh Downloading pygments-2.19.2-py3-none-any.whl (1.2 MB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 83.4 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB) | |
| oss-gpt120b-bench-94ldh Downloading python_dotenv-1.1.1-py3-none-any.whl (20 kB) | |
| oss-gpt120b-bench-94ldh Downloading pytz-2025.2-py2.py3-none-any.whl (509 kB) | |
| oss-gpt120b-bench-94ldh Downloading pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (807 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 807.9/807.9 kB 63.2 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (802 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 802.0/802.0 kB 66.9 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading rich_rst-1.3.2-py3-none-any.whl (12 kB) | |
| oss-gpt120b-bench-94ldh Downloading ruamel.yaml.clib-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (753 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 753.1/753.1 kB 80.3 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading safetensors-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (485 kB) | |
| oss-gpt120b-bench-94ldh Downloading sniffio-1.3.1-py3-none-any.whl (10 kB) | |
| oss-gpt120b-bench-94ldh Downloading tokenizers-0.22.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/3.3 MB 89.2 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading typing_extensions-4.15.0-py3-none-any.whl (44 kB) | |
| oss-gpt120b-bench-94ldh Downloading typing_inspection-0.4.2-py3-none-any.whl (14 kB) | |
| oss-gpt120b-bench-94ldh Downloading tzdata-2025.2-py2.py3-none-any.whl (347 kB) | |
| oss-gpt120b-bench-94ldh Downloading yarl-1.22.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (377 kB) | |
| oss-gpt120b-bench-94ldh Downloading filelock-3.20.0-py3-none-any.whl (16 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.2/3.2 MB 118.9 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading idna-3.11-py3-none-any.whl (71 kB) | |
| oss-gpt120b-bench-94ldh Downloading linkify_it_py-2.0.3-py3-none-any.whl (19 kB) | |
| oss-gpt120b-bench-94ldh Downloading mdit_py_plugins-0.5.0-py3-none-any.whl (57 kB) | |
| oss-gpt120b-bench-94ldh Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB) | |
| oss-gpt120b-bench-94ldh Downloading six-1.17.0-py2.py3-none-any.whl (11 kB) | |
| oss-gpt120b-bench-94ldh Downloading urllib3-2.5.0-py3-none-any.whl (129 kB) | |
| oss-gpt120b-bench-94ldh Downloading docutils-0.22.2-py3-none-any.whl (632 kB) | |
| oss-gpt120b-bench-94ldh ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 632.7/632.7 kB 62.9 MB/s eta 0:00:00 | |
| oss-gpt120b-bench-94ldh Downloading pycparser-2.23-py3-none-any.whl (118 kB) | |
| oss-gpt120b-bench-94ldh Downloading h11-0.16.0-py3-none-any.whl (37 kB) | |
| oss-gpt120b-bench-94ldh Downloading uc_micro_py-1.0.3-py3-none-any.whl (6.2 kB) | |
| oss-gpt120b-bench-94ldh Building wheels for collected packages: aiperf | |
| oss-gpt120b-bench-94ldh Building wheel for aiperf (pyproject.toml): started | |
| oss-gpt120b-bench-94ldh Building wheel for aiperf (pyproject.toml): finished with status 'done' | |
| oss-gpt120b-bench-94ldh Created wheel for aiperf: filename=aiperf-0.1.1-py3-none-any.whl size=2801033 sha256=f2f7af06bfb8c51d116627256e7dbc59e54e1e8c2c00a47474d2447935ffa1be | |
| oss-gpt120b-bench-94ldh Stored in directory: /root/.cache/pip/wheels/d7/a5/95/2d19f6967192cc77d1f77e497d7246d1b41c9decfb0c756dc6 | |
| oss-gpt120b-bench-94ldh Successfully built aiperf | |
| oss-gpt120b-bench-94ldh Installing collected packages: pytz, uvloop, urllib3, uc-micro-py, tzdata, typing-extensions, tqdm, sniffio, six, setproctitle, safetensors, ruamel.yaml.clib, regex, pyzmq, pyyaml, python-dotenv, pygments, pycparser, psutil, propcache, platformdirs, pillow, packaging, orjson, numpy, multidict, mdurl, jiter, idna, hf-xet, h11, fsspec, frozenlist, filelock, docutils, docstring-parser, distro, charset_normalizer, certifi, attrs, annotated-types, aiohappyeyeballs, aiofiles, yarl, typing-inspection, ruamel-yaml, requests, python-dateutil, pydantic-core, markdown-it-py, linkify-it-py, httpcore, cffi, anyio, aiosignal, soundfile, rich, pydantic, pandas, mdit-py-plugins, huggingface-hub, httpx, aiohttp, tokenizers, rich-rst, pydantic-settings, openai, httpx-aiohttp, transformers, textual, cyclopts, aiperf | |
| oss-gpt120b-bench-94ldh Successfully installed aiofiles-24.1.0 aiohappyeyeballs-2.6.1 aiohttp-3.12.15 aiosignal-1.4.0 aiperf-0.1.1 annotated-types-0.7.0 anyio-4.11.0 attrs-25.4.0 certifi-2025.10.5 cffi-2.0.0 charset_normalizer-3.4.4 cyclopts-3.24.0 distro-1.9.0 docstring-parser-0.17.0 docutils-0.22.2 filelock-3.20.0 frozenlist-1.8.0 fsspec-2025.9.0 h11-0.16.0 hf-xet-1.1.10 httpcore-1.0.9 httpx-0.28.1 httpx-aiohttp-0.1.8 huggingface-hub-0.35.3 idna-3.11 jiter-0.11.0 linkify-it-py-2.0.3 markdown-it-py-4.0.0 mdit-py-plugins-0.5.0 mdurl-0.1.2 multidict-6.7.0 numpy-2.2.6 openai-1.92.3 orjson-3.10.18 packaging-25.0 pandas-2.3.3 pillow-11.1.0 platformdirs-4.5.0 propcache-0.4.1 psutil-7.0.0 pycparser-2.23 pydantic-2.11.10 pydantic-core-2.33.2 pydantic-settings-2.10.1 pygments-2.19.2 python-dateutil-2.9.0.post0 python-dotenv-1.1.1 pytz-2025.2 pyyaml-6.0.3 pyzmq-26.4.0 regex-2025.9.18 requests-2.32.5 rich-14.1.0 rich-rst-1.3.2 ruamel-yaml-0.18.15 ruamel.yaml.clib-0.2.14 safetensors-0.6.2 setproctitle-1.3.7 six-1.17.0 sniffio-1.3.1 soundfile-0.13.1 textual-5.3.0 tokenizers-0.22.1 tqdm-4.67.1 transformers-4.57.1 typing-extensions-4.15.0 typing-inspection-0.4.2 tzdata-2025.2 uc-micro-py-1.0.3 urllib3-2.5.0 uvloop-0.21.0 yarl-1.22.0 | |
| oss-gpt120b-bench-94ldh WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh [notice[] A new release of pip is available: 25.0.1 -> 25.2 | |
| oss-gpt120b-bench-94ldh [notice[] To update, run: pip install --upgrade pip | |
| oss-gpt120b-bench-94ldh aiperf installation completed | |
| oss-gpt120b-bench-94ldh net.ipv4.ip_local_port_range = 1024 65000 | |
| oss-gpt120b-bench-94ldh 1024 65000 | |
| oss-gpt120b-bench-94ldh Waiting for model 'openai/gpt-oss-120b' at gpt-oss-agg-frontend:8000/v1/models (checking every 5s)... | |
| oss-gpt120b-bench-94ldh ✅ Model 'openai/gpt-oss-120b' is now available! | |
| oss-gpt120b-bench-94ldh Model 'openai/gpt-oss-120b' is now available! | |
| oss-gpt120b-bench-94ldh { | |
| oss-gpt120b-bench-94ldh "object": "list", | |
| oss-gpt120b-bench-94ldh "data": [ | |
| oss-gpt120b-bench-94ldh { | |
| oss-gpt120b-bench-94ldh "id": "openai/gpt-oss-120b", | |
| oss-gpt120b-bench-94ldh "object": "object", | |
| oss-gpt120b-bench-94ldh "created": 1760477552, | |
| oss-gpt120b-bench-94ldh "owned_by": "nvidia" | |
| oss-gpt120b-bench-94ldh } | |
| oss-gpt120b-bench-94ldh ] | |
| oss-gpt120b-bench-94ldh } | |
| oss-gpt120b-bench-94ldh Calculated total concurrency: 3600 (900 per GPU × 4 GPUs) | |
| oss-gpt120b-bench-94ldh ARTIFACT_DIR: /model-cache/perf/1760477552_oss-gpt120b-bench/concurrency_3600 | |
| oss-gpt120b-bench-94ldh 21:32:34.187347 INFO Starting AIPerf System cli_runner.py:32 | |
| oss-gpt120b-bench-94ldh 21:32:36.700445 INFO Registered Record Processor (id: 'record_processor_5a81ae97') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.711248 INFO Registered Worker Manager (id: 'worker_manager_25ec0444') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.713606 INFO Registered Timing Manager (id: 'timing_manager_2571b048') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.715719 INFO Registered Records Manager (id: 'records_manager_278a13b0') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.719300 INFO Registered Record Processor (id: 'record_processor_c526d45e') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.721111 INFO Registered Record Processor (id: 'record_processor_9d04e0f2') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.722954 INFO Registered Record Processor (id: 'record_processor_1f83c268') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.725637 INFO Registered Record Processor (id: 'record_processor_7f51ad9a') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.728653 INFO Registered Record Processor (id: 'record_processor_73c88a7f') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.731627 INFO Registered Record Processor (id: 'record_processor_a0f76e04') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.733457 INFO Registered Record Processor (id: 'record_processor_51b9a82c') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.735885 INFO Registered Record Processor (id: 'record_processor_0436d01f') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.738036 INFO Registered Record Processor (id: 'record_processor_58d4a127') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.740340 INFO Registered Record Processor (id: 'record_processor_9408bbb6') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.742156 INFO Registered Record Processor (id: 'record_processor_c84fabaa') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.744254 INFO Registered Record Processor (id: 'record_processor_0c3a84ec') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.746718 INFO Registered Record Processor (id: 'record_processor_1e8e4bdd') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.748561 INFO Registered Record Processor (id: 'record_processor_ce6bcd87') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.750872 INFO Registered Record Processor (id: 'record_processor_f6cfea80') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.752783 INFO Registered Record Processor (id: 'record_processor_8692006c') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.754690 INFO Registered Record Processor (id: 'record_processor_87314a00') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.757178 INFO Registered Record Processor (id: 'record_processor_36c3aff9') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.758931 INFO Registered Record Processor (id: 'record_processor_8ddd0342') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.761124 INFO Registered Record Processor (id: 'record_processor_596ecf4a') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.763350 INFO Registered Record Processor (id: 'record_processor_eb9bf28f') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.765138 INFO Registered Record Processor (id: 'record_processor_88a38dab') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.769356 INFO Registered Record Processor (id: 'record_processor_7f9d7b73') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:36.771331 INFO Registered Record Processor (id: 'record_processor_ec648de1') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh │ 85 │ │ return DecodeError │ | |
| oss-gpt120b-bench-94ldh │ 86 │ else: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 87 │ │ raise ImportError(PROTOBUF_IMPORT_ERROR.format(error_message)) │ | |
| oss-gpt120b-bench-94ldh │ 88 │ | |
| oss-gpt120b-bench-94ldh │ 89 │ | |
| oss-gpt120b-bench-94ldh │ 90 def flatten(arr: list): │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh ImportError: | |
| oss-gpt120b-bench-94ldh requires the protobuf library but it was not found in your environment. Check out the instructions on the | |
| oss-gpt120b-bench-94ldh installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones | |
| oss-gpt120b-bench-94ldh that match your environment. Please note that you may need to restart your runtime after installation. | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh The above exception was the direct cause of the following exception: | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:185 in run_hooks │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 182 │ │ for hook in self.get_hooks(*hook_types, reverse=reverse): │ | |
| oss-gpt120b-bench-94ldh │ 183 │ │ │ self.debug(lambda hook=hook: f"Running hook: {hook!r}") │ | |
| oss-gpt120b-bench-94ldh │ 184 │ │ │ try: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 185 │ │ │ │ await hook(**kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 186 │ │ │ except Exception as e: │ | |
| oss-gpt120b-bench-94ldh │ 187 │ │ │ │ exceptions.append(e) │ | |
| oss-gpt120b-bench-94ldh │ 188 │ │ │ │ self.exception( │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/hooks.py:114 in __call__ │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 111 │ │ | |
| oss-gpt120b-bench-94ldh │ 112 │ async def __call__(self, **kwargs) -> None: │ | |
| oss-gpt120b-bench-94ldh │ 113 │ │ if asyncio.iscoroutinefunction(self.func): │ | |
| oss-gpt120b-bench-94ldh │ ❱ 114 │ │ │ await self.func(**kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 115 │ │ else: │ | |
| oss-gpt120b-bench-94ldh │ 116 │ │ │ await asyncio.to_thread(self.func, **kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 117 │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/dataset/dataset_manager.py:80 in _initialize │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 77 │ │ │ # How will we know which tokenizer to use? │ | |
| oss-gpt120b-bench-94ldh │ 78 │ │ │ tokenizer_name = self.user_config.endpoint.model_names[0[] │ | |
| oss-gpt120b-bench-94ldh │ 79 │ │ │ | |
| oss-gpt120b-bench-94ldh │ ❱ 80 │ │ self.tokenizer = Tokenizer.from_pretrained( │ | |
| oss-gpt120b-bench-94ldh │ 81 │ │ │ tokenizer_name, │ | |
| oss-gpt120b-bench-94ldh │ 82 │ │ │ trust_remote_code=self.user_config.tokenizer.trust_remote_code, │ | |
| oss-gpt120b-bench-94ldh │ 83 │ │ │ revision=self.user_config.tokenizer.revision, │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/tokenizer.py:61 in from_pretrained │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 58 │ │ │ │ name, trust_remote_code=trust_remote_code, revision=revision │ | |
| oss-gpt120b-bench-94ldh │ 59 │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh │ 60 │ │ except Exception as e: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 61 │ │ │ raise InitializationError(e) from e │ | |
| oss-gpt120b-bench-94ldh │ 62 │ │ return tokenizer_cls │ | |
| oss-gpt120b-bench-94ldh │ 63 │ │ | |
| oss-gpt120b-bench-94ldh │ 64 │ def __call__(self, text, **kwargs) -> "BatchEncoding": │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh InitializationError: InitializationError: | |
| oss-gpt120b-bench-94ldh requires the protobuf library but it was not found in your environment. Check out the instructions on the | |
| oss-gpt120b-bench-94ldh installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones | |
| oss-gpt120b-bench-94ldh that match your environment. Please note that you may need to restart your runtime after installation. | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh 21:32:40.236531 INFO Registered Worker (id: 'worker_207e6561') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.238905 INFO Registered Worker (id: 'worker_760d56cc') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.241521 INFO Registered Worker (id: 'worker_47e1d578') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.243722 INFO Registered Worker (id: 'worker_8e8a3da6') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.246151 INFO Registered Worker (id: 'worker_c8133ed3') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.248439 INFO Registered Worker (id: 'worker_88f55eae') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.250469 INFO Registered Worker (id: 'worker_72dcb59e') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.252245 INFO Registered Worker (id: 'worker_ef344395') system_controller.py:231 | |
| oss-gpt120b-bench-94ldh 21:32:40.240633 ERROR Failed for DatasetManager (id=dataset_manager_ad73c164): AIPerfMultiError: Errors running (AIPerfHook.ON_INIT,) hooks for DatasetManager: aiperf_lifecycle_mixin.py:236 | |
| oss-gpt120b-bench-94ldh requires the protobuf library but it was not found in your environment. Check out the instructions on the | |
| oss-gpt120b-bench-94ldh installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones | |
| oss-gpt120b-bench-94ldh that match your environment. Please note that you may need to restart your runtime after installation. | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/aiperf_lifecycle_mixin.py:121 in _execute_state_transition │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 118 │ │ await self._set_state(transient_state) │ | |
| oss-gpt120b-bench-94ldh │ 119 │ │ self.debug(lambda: f"{transient_state.title()} {self}") │ | |
| oss-gpt120b-bench-94ldh │ 120 │ │ try: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 121 │ │ │ await self.run_hooks(hook_type, reverse=reverse) │ | |
| oss-gpt120b-bench-94ldh │ 122 │ │ │ await self._set_state(final_state) │ | |
| oss-gpt120b-bench-94ldh │ 123 │ │ │ self.debug(lambda: f"{self} is now {final_state.title()}") │ | |
| oss-gpt120b-bench-94ldh │ 124 │ │ │ event.set() │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:192 in run_hooks │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 189 │ │ │ │ │ f"Error running {hook!r} hook for {self.__class__.__name__}: {e}" │ | |
| oss-gpt120b-bench-94ldh │ 190 │ │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh │ 191 │ │ if exceptions: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 192 │ │ │ raise AIPerfMultiError( │ | |
| oss-gpt120b-bench-94ldh │ 193 │ │ │ │ f"Errors running {hook_types} hooks for {self.__class__.__name__}", │ | |
| oss-gpt120b-bench-94ldh │ 194 │ │ │ │ exceptions, │ | |
| oss-gpt120b-bench-94ldh │ 195 │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh AIPerfMultiError: AIPerfMultiError: Errors running (AIPerfHook.ON_INIT,) hooks for DatasetManager: | |
| oss-gpt120b-bench-94ldh requires the protobuf library but it was not found in your environment. Check out the instructions on the | |
| oss-gpt120b-bench-94ldh installation page of its repo: https://github.com/protocolbuffers/protobuf/tree/master/python#installation and follow the ones | |
| oss-gpt120b-bench-94ldh that match your environment. Please note that you may need to restart your runtime after installation. | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh 21:33:06.698968 ERROR Service dataset_manager failed to register within timeout multiprocess_service_manager.py:190 | |
| oss-gpt120b-bench-94ldh │ 175 │ │ │ │ await asyncio.sleep(0.5) │ | |
| oss-gpt120b-bench-94ldh │ 176 │ │ │ | |
| oss-gpt120b-bench-94ldh │ 177 │ │ try: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 178 │ │ │ await asyncio.wait_for(_wait_for_registration(), timeout=timeout_seconds) │ | |
| oss-gpt120b-bench-94ldh │ 179 │ │ except asyncio.TimeoutError as e: │ | |
| oss-gpt120b-bench-94ldh │ 180 │ │ │ # Log which services didn't register in time │ | |
| oss-gpt120b-bench-94ldh │ 181 │ │ │ registered_types_set = set( │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/asyncio/tasks.py:519 in wait_for │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 516 │ │ except exceptions.CancelledError as exc: │ | |
| oss-gpt120b-bench-94ldh │ 517 │ │ │ raise TimeoutError from exc │ | |
| oss-gpt120b-bench-94ldh │ 518 │ │ | |
| oss-gpt120b-bench-94ldh │ ❱ 519 │ async with timeouts.timeout(timeout): │ | |
| oss-gpt120b-bench-94ldh │ 520 │ │ return await fut │ | |
| oss-gpt120b-bench-94ldh │ 521 │ | |
| oss-gpt120b-bench-94ldh │ 522 async def _wait(fs, timeout, return_when, loop): │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/asyncio/timeouts.py:115 in __aexit__ │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 112 │ │ │ if self._task.uncancel() <= self._cancelling and exc_type is │ | |
| oss-gpt120b-bench-94ldh │ exceptions.CancelledError: │ | |
| oss-gpt120b-bench-94ldh │ 113 │ │ │ │ # Since there are no new cancel requests, we're │ | |
| oss-gpt120b-bench-94ldh │ 114 │ │ │ │ # handling this. │ | |
| oss-gpt120b-bench-94ldh │ ❱ 115 │ │ │ │ raise TimeoutError from exc_val │ | |
| oss-gpt120b-bench-94ldh │ 116 │ │ elif self._state is _State.ENTERED: │ | |
| oss-gpt120b-bench-94ldh │ 117 │ │ │ self._state = _State.EXITED │ | |
| oss-gpt120b-bench-94ldh │ 118 │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh TimeoutError | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh The above exception was the direct cause of the following exception: | |
| oss-gpt120b-bench-94ldh | |
| oss-gpt120b-bench-94ldh ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:185 in run_hooks │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 182 │ │ for hook in self.get_hooks(*hook_types, reverse=reverse): │ | |
| oss-gpt120b-bench-94ldh │ 183 │ │ │ self.debug(lambda hook=hook: f"Running hook: {hook!r}") │ | |
| oss-gpt120b-bench-94ldh │ 184 │ │ │ try: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 185 │ │ │ │ await hook(**kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 186 │ │ │ except Exception as e: │ | |
| oss-gpt120b-bench-94ldh │ 187 │ │ │ │ exceptions.append(e) │ | |
| oss-gpt120b-bench-94ldh │ 188 │ │ │ │ self.exception( │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/hooks.py:114 in __call__ │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 111 │ │ | |
| oss-gpt120b-bench-94ldh │ 112 │ async def __call__(self, **kwargs) -> None: │ | |
| oss-gpt120b-bench-94ldh │ 113 │ │ if asyncio.iscoroutinefunction(self.func): │ | |
| oss-gpt120b-bench-94ldh │ ❱ 114 │ │ │ await self.func(**kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 115 │ │ else: │ | |
| oss-gpt120b-bench-94ldh │ 116 │ │ │ await asyncio.to_thread(self.func, **kwargs) │ | |
| oss-gpt120b-bench-94ldh │ 117 │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/controller/system_controller.py:156 in _start_services │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 153 │ │ self.debug("System Controller is bootstrapping services") │ | |
| oss-gpt120b-bench-94ldh │ 154 │ │ # Start all required services │ | |
| oss-gpt120b-bench-94ldh │ 155 │ │ await self.service_manager.start() │ | |
| oss-gpt120b-bench-94ldh │ ❱ 156 │ │ await self.service_manager.wait_for_all_services_registration( │ | |
| oss-gpt120b-bench-94ldh │ 157 │ │ │ stop_event=self._stop_requested_event, │ | |
| oss-gpt120b-bench-94ldh │ 158 │ │ ) │ | |
| oss-gpt120b-bench-94ldh │ 159 │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/controller/multiprocess_service_manager.py:194 in wait_for_all_services_registration │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 191 │ │ │ │ │ │ f"Service {service_type} failed to register within timeout" │ | |
| oss-gpt120b-bench-94ldh │ 192 │ │ │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh │ 193 │ │ │ │ | |
| oss-gpt120b-bench-94ldh │ ❱ 194 │ │ │ raise AIPerfError("Some services failed to register within timeout") from e │ | |
| oss-gpt120b-bench-94ldh │ 195 │ │ | |
| oss-gpt120b-bench-94ldh │ 196 │ async def _wait_for_process(self, info: MultiProcessRunInfo) -> None: │ | |
| oss-gpt120b-bench-94ldh │ 197 │ │ """Wait for a process to terminate with timeout handling.""" │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh AIPerfError: AIPerfError: Some services failed to register within timeout | |
| oss-gpt120b-bench-94ldh 21:33:07.068016 ERROR Failed for SystemController (id=system_controller): AIPerfMultiError: Errors running (AIPerfHook.ON_START,) hooks for SystemController: Some aiperf_lifecycle_mixin.py:236 | |
| oss-gpt120b-bench-94ldh services failed to register within timeout | |
| oss-gpt120b-bench-94ldh ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/aiperf_lifecycle_mixin.py:121 in _execute_state_transition │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 118 │ │ await self._set_state(transient_state) │ | |
| oss-gpt120b-bench-94ldh │ 119 │ │ self.debug(lambda: f"{transient_state.title()} {self}") │ | |
| oss-gpt120b-bench-94ldh │ 120 │ │ try: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 121 │ │ │ await self.run_hooks(hook_type, reverse=reverse) │ | |
| oss-gpt120b-bench-94ldh │ 122 │ │ │ await self._set_state(final_state) │ | |
| oss-gpt120b-bench-94ldh │ 123 │ │ │ self.debug(lambda: f"{self} is now {final_state.title()}") │ | |
| oss-gpt120b-bench-94ldh │ 124 │ │ │ event.set() │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:192 in run_hooks │ | |
| oss-gpt120b-bench-94ldh │ │ | |
| oss-gpt120b-bench-94ldh │ 189 │ │ │ │ │ f"Error running {hook!r} hook for {self.__class__.__name__}: {e}" │ | |
| oss-gpt120b-bench-94ldh │ 190 │ │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh │ 191 │ │ if exceptions: │ | |
| oss-gpt120b-bench-94ldh │ ❱ 192 │ │ │ raise AIPerfMultiError( │ | |
| oss-gpt120b-bench-94ldh │ 193 │ │ │ │ f"Errors running {hook_types} hooks for {self.__class__.__name__}", │ | |
| oss-gpt120b-bench-94ldh │ 194 │ │ │ │ exceptions, │ | |
| oss-gpt120b-bench-94ldh │ 195 │ │ │ ) │ | |
| oss-gpt120b-bench-94ldh ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| oss-gpt120b-bench-94ldh AIPerfMultiError: AIPerfMultiError: Errors running (AIPerfHook.ON_START,) hooks for SystemController: Some services failed to register within | |
| oss-gpt120b-bench-94ldh timeout |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment