Merge pull request 'fix: retry transient RM allocation failures so power control block doesn't vanish' (#20) from fix/rm-power-transient-allocation into main

Reviewed-on: #20
This commit was merged in pull request #20.
This commit is contained in:
Pakobbix committed 2026-11-07 14:05:26 +00:00
commit 8567eee475
3 files changed
+137 -4

No files matched your search

@@ -1,4 +1,4 @@
import { useState, useEffect } from "react";
import { useState, useEffect, useRef } from "react";
import { Check, X, RotateCcw } from "lucide-react";
import { api } from "../../api/client.js";
import { useCurveStore } from "../../store/curveStore.js";
@@ -17,15 +17,45 @@ export function PerformancePanel() {
const [error, setError] = useState<string | null>(null);
const [confirmApply, setConfirmApply] = useState(false);
const [confirmReset, setConfirmReset] = useState(false);
// Monotonic fetch counter: a response may only commit state (or toast /
// clear loading) if it is still the latest fetch for this panel, so a slow
// response for a previously selected GPU cannot clobber fresher results.
const fetchSeq = useRef(0);
async function fetchLimits() {
const seq = ++fetchSeq.current;
try {
setLoading(true);
setLimits(await api.limits(selectedGpuIndex));
let next = await api.limits(selectedGpuIndex);
// The RM support probe can flake transiently (the driver briefly
// rejects RM object allocation when busy) and report
// rm_power_supported=false for one read. A single failed read must not
// make the Experimental Power Control block vanish — or flip it into
// "interface unavailable" — so confirm a negative reading with one
// follow-up fetch before trusting it. The confirm has its own error
// boundary: if it throws (network blip), keep the first result, which
// is still a valid read, rather than discarding it into the outer catch.
if (!next.rm_power_supported) {
try {
const again = await api.limits(selectedGpuIndex);
if (again.rm_power_supported) {
next = again;
}
} catch {
// keep first result
}
}
if (seq === fetchSeq.current) {
setLimits(next);
}
} catch {
toast.error("Failed to load performance limits");
if (seq === fetchSeq.current) {
toast.error("Failed to load performance limits");
}
} finally {
setLoading(false);
if (seq === fetchSeq.current) {
setLoading(false);
}
}
}
+29
View File
@@ -20,6 +20,7 @@ import logging
import os
import struct
import sys
import time
from collections.abc import Callable
from dataclasses import dataclass
@@ -343,8 +344,36 @@ class RmHandle:
self.device_handle = device_handle
self.subdevice_handle = subdevice_handle
# Transient RM allocation failures (observed: NV0080 alloc rejected with
# status 0x1f) happen when GSP is still busy with a preceding operation —
# e.g. the first probe in a fresh process, or a request issued directly
# after a power-limit write. The allocation succeeds on immediate
# re-attempt, so retry a short, bounded number of times before reporting
# the interface as unavailable.
_ALLOCATION_ATTEMPTS = 3
_ALLOCATION_RETRY_DELAY_S = 0.1
@classmethod
def open(cls, gpu_index: int) -> RmHandle:
"""Open an RM handle, retrying transient driver allocation failures."""
last_error: RmPowerError | None = None
for attempt in range(cls._ALLOCATION_ATTEMPTS):
try:
return cls._open_once(gpu_index)
except RmPowerError as exc:
last_error = exc
if attempt + 1 < cls._ALLOCATION_ATTEMPTS:
log.debug(
"RM handle open failed (attempt %d/%d): %s",
attempt + 1,
cls._ALLOCATION_ATTEMPTS,
exc,
)
time.sleep(cls._ALLOCATION_RETRY_DELAY_S)
raise last_error or RmPowerError("RM handle open failed")
@classmethod
def _open_once(cls, gpu_index: int) -> RmHandle:
"""Open an RM handle for the GPU at the given NVML index.
The RM device/subdevice instances are resolved by PCI identity
+74
View File
@@ -320,6 +320,78 @@ def test_rejects_nonzero_pci_function() -> None:
check("nonzero function rejected", True)
# ── Transient RM allocation retries (RmHandle.open) ─────────────────────────
def _patch_open_once(fake) -> tuple:
from nvcurve.hal import rm_power as _rp
orig = _rp.RmHandle._open_once
orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S
_rp.RmHandle._open_once = fake
_rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0
return orig, orig_delay
def test_handle_open_retries_transient_allocation_failure() -> None:
"""GSP rejects RM object allocation with status 0x1f when busy (first
probe in a fresh process, or a request right after a power-limit write);
it succeeds on immediate re-attempt. A single transient rejection must
not surface as "interface not supported"."""
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
sentinel = object()
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS:
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
return sentinel
orig, orig_delay = _patch_open_once(fake)
try:
handle = rp.RmHandle.open(0)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check("transient failure retried until success", handle is sentinel)
check(
"succeeds on the final allowed attempt",
attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_handle_open_reports_persistent_allocation_failure() -> None:
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
orig, orig_delay = _patch_open_once(fake)
try:
try:
rp.RmHandle.open(0)
check("persistent failure still raises", False)
except RmPowerError as exc:
check(
"persistent failure still raises with the original error",
"status 0x1f" in str(exc),
)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check(
"gives up after the bounded attempt count",
len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_propagates_rm_query_failure() -> None:
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
try:
@@ -342,6 +414,8 @@ def main() -> int:
test_resolves_pci_identity_when_minor_and_rm_orders_differ,
test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
test_rejects_nonzero_pci_function,
test_handle_open_retries_transient_allocation_failure,
test_handle_open_reports_persistent_allocation_failure,
test_propagates_rm_query_failure,
]
for t in tests: