fix: retry transient RM allocation failures so power control block doesn't vanish

GSP transiently rejects NV0080 object allocation with status 0x1f when
busy — reliably on the first probe in a fresh server process and
intermittently right after RM operations (e.g. a power-limit write).
The probe surfaced that transient rejection as rm_power_supported=false,
making the Experimental Power Control block disappear in the UI until a
reload re-probed successfully.

- hal/rm_power: RmHandle.open() now retries the handle allocation
  (3 attempts, 100 ms apart) before reporting the interface unavailable;
  covers probe, set, and enable-check paths. Each attempt cleans up its
  own fds, so retries are safe and idempotent.
- PerformancePanel: confirm a negative rm_power_supported reading with
  one follow-up fetch before rendering it, so a single flaky probe can't
  hide the block or flip it into "interface unavailable"; the confirm
  has its own error boundary so a failed second request keeps the first
  (valid) read. Also added a monotonic fetch-seq guard so a slow
  response for a previously selected GPU cannot clobber fresher state.
- tests: cover transient-failure-retry and persistent-failure paths for
  RmHandle.open.

Verified live: first probe after restart 5/5 (was 0/3), 150 rapid
set+get cycles with 0 probe flips and 0 set failures.
This commit is contained in:
ARIA committed 2026-11-07 14:41:35 +01:00
1 parent 6afdd15d2a
commit 2e8e580d79
3 files changed
+137 -4

No files matched your search

+74
View File
@@ -320,6 +320,78 @@ def test_rejects_nonzero_pci_function() -> None:
check("nonzero function rejected", True)
# ── Transient RM allocation retries (RmHandle.open) ─────────────────────────
def _patch_open_once(fake) -> tuple:
from nvcurve.hal import rm_power as _rp
orig = _rp.RmHandle._open_once
orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S
_rp.RmHandle._open_once = fake
_rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0
return orig, orig_delay
def test_handle_open_retries_transient_allocation_failure() -> None:
"""GSP rejects RM object allocation with status 0x1f when busy (first
probe in a fresh process, or a request right after a power-limit write);
it succeeds on immediate re-attempt. A single transient rejection must
not surface as "interface not supported"."""
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
sentinel = object()
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS:
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
return sentinel
orig, orig_delay = _patch_open_once(fake)
try:
handle = rp.RmHandle.open(0)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check("transient failure retried until success", handle is sentinel)
check(
"succeeds on the final allowed attempt",
attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_handle_open_reports_persistent_allocation_failure() -> None:
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
orig, orig_delay = _patch_open_once(fake)
try:
try:
rp.RmHandle.open(0)
check("persistent failure still raises", False)
except RmPowerError as exc:
check(
"persistent failure still raises with the original error",
"status 0x1f" in str(exc),
)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check(
"gives up after the bounded attempt count",
len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_propagates_rm_query_failure() -> None:
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
try:
@@ -342,6 +414,8 @@ def main() -> int:
test_resolves_pci_identity_when_minor_and_rm_orders_differ,
test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
test_rejects_nonzero_pci_function,
test_handle_open_retries_transient_allocation_failure,
test_handle_open_reports_persistent_allocation_failure,
test_propagates_rm_query_failure,
]
for t in tests: