Created
October 14, 2025 21:16
-
-
Save sozercan/1f0fb725b80fa54be1adc11dde1d212c to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| │ 175 │ │ │ │ await asyncio.sleep(0.5) │ | |
| │ 176 │ │ │ | |
| │ 177 │ │ try: │ | |
| │ ❱ 178 │ │ │ await asyncio.wait_for(_wait_for_registration(), timeout=timeout_seconds) │ | |
| │ 179 │ │ except asyncio.TimeoutError as e: │ | |
| │ 180 │ │ │ # Log which services didn't register in time │ | |
| │ 181 │ │ │ registered_types_set = set( │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/asyncio/tasks.py:519 in wait_for │ | |
| │ │ | |
| │ 516 │ │ except exceptions.CancelledError as exc: │ | |
| │ 517 │ │ │ raise TimeoutError from exc │ | |
| │ 518 │ │ | |
| │ ❱ 519 │ async with timeouts.timeout(timeout): │ | |
| │ 520 │ │ return await fut │ | |
| │ 521 │ | |
| │ 522 async def _wait(fs, timeout, return_when, loop): │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/asyncio/timeouts.py:115 in __aexit__ │ | |
| │ │ | |
| │ 112 │ │ │ if self._task.uncancel() <= self._cancelling and exc_type is │ | |
| │ exceptions.CancelledError: │ | |
| │ 113 │ │ │ │ # Since there are no new cancel requests, we're │ | |
| │ 114 │ │ │ │ # handling this. │ | |
| │ ❱ 115 │ │ │ │ raise TimeoutError from exc_val │ | |
| │ 116 │ │ elif self._state is _State.ENTERED: │ | |
| │ 117 │ │ │ self._state = _State.EXITED │ | |
| │ 118 │ | |
| ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| TimeoutError | |
| The above exception was the direct cause of the following exception: | |
| ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:185 in run_hooks │ | |
| │ │ | |
| │ 182 │ │ for hook in self.get_hooks(*hook_types, reverse=reverse): │ | |
| │ 183 │ │ │ self.debug(lambda hook=hook: f"Running hook: {hook!r}") │ | |
| │ 184 │ │ │ try: │ | |
| │ ❱ 185 │ │ │ │ await hook(**kwargs) │ | |
| │ 186 │ │ │ except Exception as e: │ | |
| │ 187 │ │ │ │ exceptions.append(e) │ | |
| │ 188 │ │ │ │ self.exception( │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/common/hooks.py:114 in __call__ │ | |
| │ │ | |
| │ 111 │ │ | |
| │ 112 │ async def __call__(self, **kwargs) -> None: │ | |
| │ 113 │ │ if asyncio.iscoroutinefunction(self.func): │ | |
| │ ❱ 114 │ │ │ await self.func(**kwargs) │ | |
| │ 115 │ │ else: │ | |
| │ 116 │ │ │ await asyncio.to_thread(self.func, **kwargs) │ | |
| │ 117 │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/controller/system_controller.py:156 in _start_services │ | |
| │ │ | |
| │ 153 │ │ self.debug("System Controller is bootstrapping services") │ | |
| │ 154 │ │ # Start all required services │ | |
| │ 155 │ │ await self.service_manager.start() │ | |
| │ ❱ 156 │ │ await self.service_manager.wait_for_all_services_registration( │ | |
| │ 157 │ │ │ stop_event=self._stop_requested_event, │ | |
| │ 158 │ │ ) │ | |
| │ 159 │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/controller/multiprocess_service_manager.py:194 in wait_for_all_services_registration │ | |
| │ │ | |
| │ 191 │ │ │ │ │ │ f"Service {service_type} failed to register within timeout" │ | |
| │ 192 │ │ │ │ │ ) │ | |
| │ 193 │ │ │ │ | |
| │ ❱ 194 │ │ │ raise AIPerfError("Some services failed to register within timeout") from e │ | |
| │ 195 │ │ | |
| │ 196 │ async def _wait_for_process(self, info: MultiProcessRunInfo) -> None: │ | |
| │ 197 │ │ """Wait for a process to terminate with timeout handling.""" │ | |
| ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| AIPerfError: AIPerfError: Some services failed to register within timeout | |
| 21:15:19.042554 ERROR Failed for SystemController (id=system_controller): AIPerfMultiError: Errors running (AIPerfHook.ON_START,) hooks for SystemController: Some aiperf_lifecycle_mixin.py:236 | |
| services failed to register within timeout | |
| ╭────────────────────────────────────────────────────── Traceback (most recent call last) ──────────────────────────────────────────────────────╮ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/aiperf_lifecycle_mixin.py:121 in _execute_state_transition │ | |
| │ │ | |
| │ 118 │ │ await self._set_state(transient_state) │ | |
| │ 119 │ │ self.debug(lambda: f"{transient_state.title()} {self}") │ | |
| │ 120 │ │ try: │ | |
| │ ❱ 121 │ │ │ await self.run_hooks(hook_type, reverse=reverse) │ | |
| │ 122 │ │ │ await self._set_state(final_state) │ | |
| │ 123 │ │ │ self.debug(lambda: f"{self} is now {final_state.title()}") │ | |
| │ 124 │ │ │ event.set() │ | |
| │ │ | |
| │ /usr/local/lib/python3.12/site-packages/aiperf/common/mixins/hooks_mixin.py:192 in run_hooks │ | |
| │ │ | |
| │ 189 │ │ │ │ │ f"Error running {hook!r} hook for {self.__class__.__name__}: {e}" │ | |
| │ 190 │ │ │ │ ) │ | |
| │ 191 │ │ if exceptions: │ | |
| │ ❱ 192 │ │ │ raise AIPerfMultiError( │ | |
| │ 193 │ │ │ │ f"Errors running {hook_types} hooks for {self.__class__.__name__}", │ | |
| │ 194 │ │ │ │ exceptions, │ | |
| │ 195 │ │ │ ) │ | |
| ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ | |
| AIPerfMultiError: AIPerfMultiError: Errors running (AIPerfHook.ON_START,) hooks for SystemController: Some services failed to register within | |
| timeout |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment