diff --git a/nvcurve/hal/gpu.py b/nvcurve/hal/gpu.py index 5e9b1ca..d206ab7 100644 --- a/nvcurve/hal/gpu.py +++ b/nvcurve/hal/gpu.py @@ -12,8 +12,7 @@ def init_nvapi() -> None: """Initialize NvAPI. Must be called before any GPU operations.""" init_fn = query_interface(FUNC["Initialize"], nargs=0) if not init_fn or init_fn() != 0: - print("NvAPI_Initialize failed") - sys.exit(1) + raise RuntimeError("NvAPI_Initialize failed") def enumerate_gpus() -> tuple[ctypes.Array, int]: diff --git a/nvcurve/profiles/apply.py b/nvcurve/profiles/apply.py index f5c1031..a3068c6 100644 --- a/nvcurve/profiles/apply.py +++ b/nvcurve/profiles/apply.py @@ -162,10 +162,18 @@ def run_autoload() -> None: from ..hal.gpu import init_nvapi, discover_gpus from ..hal.monitoring import init_nvml, shutdown_nvml - try: - init_nvapi() - except Exception as exc: - log.error("Failed to initialize NvAPI: %s", exc) + # Retry NvAPI init — the driver may not be fully ready at early boot. + nvapi_ready = False + for attempt in range(1, 17): + try: + init_nvapi() + nvapi_ready = True + break + except Exception as exc: + log.warning("NvAPI init attempt %d/16 failed: %s", attempt, exc) + time.sleep(2) + if not nvapi_ready: + log.error("Failed to initialize NvAPI after 16 attempts.") sys.exit(1) init_nvml() # best-effort