diff --git a/frontend/src/components/Limits/PerformancePanel.tsx b/frontend/src/components/Limits/PerformancePanel.tsx index 82b69d9..a0a5d63 100644 --- a/frontend/src/components/Limits/PerformancePanel.tsx +++ b/frontend/src/components/Limits/PerformancePanel.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect } from "react"; +import { useState, useEffect, useRef } from "react"; import { Check, X, RotateCcw } from "lucide-react"; import { api } from "../../api/client.js"; import { useCurveStore } from "../../store/curveStore.js"; @@ -17,15 +17,45 @@ export function PerformancePanel() { const [error, setError] = useState(null); const [confirmApply, setConfirmApply] = useState(false); const [confirmReset, setConfirmReset] = useState(false); + // Monotonic fetch counter: a response may only commit state (or toast / + // clear loading) if it is still the latest fetch for this panel, so a slow + // response for a previously selected GPU cannot clobber fresher results. + const fetchSeq = useRef(0); async function fetchLimits() { + const seq = ++fetchSeq.current; try { setLoading(true); - setLimits(await api.limits(selectedGpuIndex)); + let next = await api.limits(selectedGpuIndex); + // The RM support probe can flake transiently (the driver briefly + // rejects RM object allocation when busy) and report + // rm_power_supported=false for one read. A single failed read must not + // make the Experimental Power Control block vanish — or flip it into + // "interface unavailable" — so confirm a negative reading with one + // follow-up fetch before trusting it. The confirm has its own error + // boundary: if it throws (network blip), keep the first result, which + // is still a valid read, rather than discarding it into the outer catch. + if (!next.rm_power_supported) { + try { + const again = await api.limits(selectedGpuIndex); + if (again.rm_power_supported) { + next = again; + } + } catch { + // keep first result + } + } + if (seq === fetchSeq.current) { + setLimits(next); + } } catch { - toast.error("Failed to load performance limits"); + if (seq === fetchSeq.current) { + toast.error("Failed to load performance limits"); + } } finally { - setLoading(false); + if (seq === fetchSeq.current) { + setLoading(false); + } } } diff --git a/nvcurve/hal/rm_power.py b/nvcurve/hal/rm_power.py index 3a8f9d5..9b0a1c3 100644 --- a/nvcurve/hal/rm_power.py +++ b/nvcurve/hal/rm_power.py @@ -20,6 +20,7 @@ import logging import os import struct import sys +import time from collections.abc import Callable from dataclasses import dataclass @@ -343,8 +344,36 @@ class RmHandle: self.device_handle = device_handle self.subdevice_handle = subdevice_handle + # Transient RM allocation failures (observed: NV0080 alloc rejected with + # status 0x1f) happen when GSP is still busy with a preceding operation — + # e.g. the first probe in a fresh process, or a request issued directly + # after a power-limit write. The allocation succeeds on immediate + # re-attempt, so retry a short, bounded number of times before reporting + # the interface as unavailable. + _ALLOCATION_ATTEMPTS = 3 + _ALLOCATION_RETRY_DELAY_S = 0.1 + @classmethod def open(cls, gpu_index: int) -> RmHandle: + """Open an RM handle, retrying transient driver allocation failures.""" + last_error: RmPowerError | None = None + for attempt in range(cls._ALLOCATION_ATTEMPTS): + try: + return cls._open_once(gpu_index) + except RmPowerError as exc: + last_error = exc + if attempt + 1 < cls._ALLOCATION_ATTEMPTS: + log.debug( + "RM handle open failed (attempt %d/%d): %s", + attempt + 1, + cls._ALLOCATION_ATTEMPTS, + exc, + ) + time.sleep(cls._ALLOCATION_RETRY_DELAY_S) + raise last_error or RmPowerError("RM handle open failed") + + @classmethod + def _open_once(cls, gpu_index: int) -> RmHandle: """Open an RM handle for the GPU at the given NVML index. The RM device/subdevice instances are resolved by PCI identity diff --git a/tests/test_rm_power.py b/tests/test_rm_power.py index 2541665..891072b 100644 --- a/tests/test_rm_power.py +++ b/tests/test_rm_power.py @@ -320,6 +320,78 @@ def test_rejects_nonzero_pci_function() -> None: check("nonzero function rejected", True) +# ── Transient RM allocation retries (RmHandle.open) ───────────────────────── + + +def _patch_open_once(fake) -> tuple: + from nvcurve.hal import rm_power as _rp + + orig = _rp.RmHandle._open_once + orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S + _rp.RmHandle._open_once = fake + _rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0 + return orig, orig_delay + + +def test_handle_open_retries_transient_allocation_failure() -> None: + """GSP rejects RM object allocation with status 0x1f when busy (first + probe in a fresh process, or a request right after a power-limit write); + it succeeds on immediate re-attempt. A single transient rejection must + not surface as "interface not supported".""" + from nvcurve.hal import rm_power as rp + + attempts: list[int] = [] + sentinel = object() + + @classmethod + def fake(klass, gpu_index: int): + attempts.append(gpu_index) + if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS: + raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)") + return sentinel + + orig, orig_delay = _patch_open_once(fake) + try: + handle = rp.RmHandle.open(0) + finally: + rp.RmHandle._open_once = orig + rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay + check("transient failure retried until success", handle is sentinel) + check( + "succeeds on the final allowed attempt", + attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS, + ) + + +def test_handle_open_reports_persistent_allocation_failure() -> None: + from nvcurve.hal import rm_power as rp + + attempts: list[int] = [] + + @classmethod + def fake(klass, gpu_index: int): + attempts.append(gpu_index) + raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)") + + orig, orig_delay = _patch_open_once(fake) + try: + try: + rp.RmHandle.open(0) + check("persistent failure still raises", False) + except RmPowerError as exc: + check( + "persistent failure still raises with the original error", + "status 0x1f" in str(exc), + ) + finally: + rp.RmHandle._open_once = orig + rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay + check( + "gives up after the bounded attempt count", + len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS, + ) + + def test_propagates_rm_query_failure() -> None: pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0) try: @@ -342,6 +414,8 @@ def main() -> int: test_resolves_pci_identity_when_minor_and_rm_orders_differ, test_does_not_fall_back_to_another_gpu_when_pci_is_missing, test_rejects_nonzero_pci_function, + test_handle_open_retries_transient_allocation_failure, + test_handle_open_reports_persistent_allocation_failure, test_propagates_rm_query_failure, ] for t in tests: