fix: retry NvAPI init on autoload to handle early boot race #3

Merged
Pakobbix merged 1 commits from fix/autoload-nvapi-retry into main 2026-08-09 06:29:59 +00:00
2 changed files with 13 additions and 6 deletions

No files matched your search

+1 -2
View File
@@ -12,8 +12,7 @@ def init_nvapi() -> None:
"""Initialize NvAPI. Must be called before any GPU operations.""" """Initialize NvAPI. Must be called before any GPU operations."""
init_fn = query_interface(FUNC["Initialize"], nargs=0) init_fn = query_interface(FUNC["Initialize"], nargs=0)
if not init_fn or init_fn() != 0: if not init_fn or init_fn() != 0:
print("NvAPI_Initialize failed") raise RuntimeError("NvAPI_Initialize failed")
sys.exit(1)
def enumerate_gpus() -> tuple[ctypes.Array, int]: def enumerate_gpus() -> tuple[ctypes.Array, int]:
+12 -4
View File
@@ -162,10 +162,18 @@ def run_autoload() -> None:
from ..hal.gpu import init_nvapi, discover_gpus from ..hal.gpu import init_nvapi, discover_gpus
from ..hal.monitoring import init_nvml, shutdown_nvml from ..hal.monitoring import init_nvml, shutdown_nvml
try: # Retry NvAPI init — the driver may not be fully ready at early boot.
init_nvapi() nvapi_ready = False
except Exception as exc: for attempt in range(1, 17):
log.error("Failed to initialize NvAPI: %s", exc) try:
init_nvapi()
nvapi_ready = True
break
except Exception as exc:
log.warning("NvAPI init attempt %d/16 failed: %s", attempt, exc)
time.sleep(2)
if not nvapi_ready:
log.error("Failed to initialize NvAPI after 16 attempts.")
sys.exit(1) sys.exit(1)
init_nvml() # best-effort init_nvml() # best-effort