fix: retry transient RM allocation failures so power control block doesn't vanish #20

Merged
Pakobbix merged 1 commits from fix/rm-power-transient-allocation into main 2026-11-07 14:05:27 +00:00
3 changed files with 135 additions and 2 deletions

No files matched your search

@@ -1,4 +1,4 @@
import { useState, useEffect } from "react"; import { useState, useEffect, useRef } from "react";
import { Check, X, RotateCcw } from "lucide-react"; import { Check, X, RotateCcw } from "lucide-react";
import { api } from "../../api/client.js"; import { api } from "../../api/client.js";
import { useCurveStore } from "../../store/curveStore.js"; import { useCurveStore } from "../../store/curveStore.js";
@@ -17,17 +17,47 @@ export function PerformancePanel() {
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [confirmApply, setConfirmApply] = useState(false); const [confirmApply, setConfirmApply] = useState(false);
const [confirmReset, setConfirmReset] = useState(false); const [confirmReset, setConfirmReset] = useState(false);
// Monotonic fetch counter: a response may only commit state (or toast /
// clear loading) if it is still the latest fetch for this panel, so a slow
// response for a previously selected GPU cannot clobber fresher results.
const fetchSeq = useRef(0);
async function fetchLimits() { async function fetchLimits() {
const seq = ++fetchSeq.current;
try { try {
setLoading(true); setLoading(true);
setLimits(await api.limits(selectedGpuIndex)); let next = await api.limits(selectedGpuIndex);
// The RM support probe can flake transiently (the driver briefly
// rejects RM object allocation when busy) and report
// rm_power_supported=false for one read. A single failed read must not
// make the Experimental Power Control block vanish — or flip it into
// "interface unavailable" — so confirm a negative reading with one
// follow-up fetch before trusting it. The confirm has its own error
// boundary: if it throws (network blip), keep the first result, which
// is still a valid read, rather than discarding it into the outer catch.
if (!next.rm_power_supported) {
try {
const again = await api.limits(selectedGpuIndex);
if (again.rm_power_supported) {
next = again;
}
} catch { } catch {
// keep first result
}
}
if (seq === fetchSeq.current) {
setLimits(next);
}
} catch {
if (seq === fetchSeq.current) {
toast.error("Failed to load performance limits"); toast.error("Failed to load performance limits");
}
} finally { } finally {
if (seq === fetchSeq.current) {
setLoading(false); setLoading(false);
} }
} }
}
// Data fetch on GPU change — setState calls happen after the await, not // Data fetch on GPU change — setState calls happen after the await, not
// synchronously in the effect body (rule false-positive on async fetch). // synchronously in the effect body (rule false-positive on async fetch).
+29
View File
@@ -20,6 +20,7 @@ import logging
import os import os
import struct import struct
import sys import sys
import time
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
@@ -343,8 +344,36 @@ class RmHandle:
self.device_handle = device_handle self.device_handle = device_handle
self.subdevice_handle = subdevice_handle self.subdevice_handle = subdevice_handle
# Transient RM allocation failures (observed: NV0080 alloc rejected with
# status 0x1f) happen when GSP is still busy with a preceding operation —
# e.g. the first probe in a fresh process, or a request issued directly
# after a power-limit write. The allocation succeeds on immediate
# re-attempt, so retry a short, bounded number of times before reporting
# the interface as unavailable.
_ALLOCATION_ATTEMPTS = 3
_ALLOCATION_RETRY_DELAY_S = 0.1
@classmethod @classmethod
def open(cls, gpu_index: int) -> RmHandle: def open(cls, gpu_index: int) -> RmHandle:
"""Open an RM handle, retrying transient driver allocation failures."""
last_error: RmPowerError | None = None
for attempt in range(cls._ALLOCATION_ATTEMPTS):
try:
return cls._open_once(gpu_index)
except RmPowerError as exc:
last_error = exc
if attempt + 1 < cls._ALLOCATION_ATTEMPTS:
log.debug(
"RM handle open failed (attempt %d/%d): %s",
attempt + 1,
cls._ALLOCATION_ATTEMPTS,
exc,
)
time.sleep(cls._ALLOCATION_RETRY_DELAY_S)
raise last_error or RmPowerError("RM handle open failed")
@classmethod
def _open_once(cls, gpu_index: int) -> RmHandle:
"""Open an RM handle for the GPU at the given NVML index. """Open an RM handle for the GPU at the given NVML index.
The RM device/subdevice instances are resolved by PCI identity The RM device/subdevice instances are resolved by PCI identity
+74
View File
@@ -320,6 +320,78 @@ def test_rejects_nonzero_pci_function() -> None:
check("nonzero function rejected", True) check("nonzero function rejected", True)
# ── Transient RM allocation retries (RmHandle.open) ─────────────────────────
def _patch_open_once(fake) -> tuple:
from nvcurve.hal import rm_power as _rp
orig = _rp.RmHandle._open_once
orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S
_rp.RmHandle._open_once = fake
_rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0
return orig, orig_delay
def test_handle_open_retries_transient_allocation_failure() -> None:
"""GSP rejects RM object allocation with status 0x1f when busy (first
probe in a fresh process, or a request right after a power-limit write);
it succeeds on immediate re-attempt. A single transient rejection must
not surface as "interface not supported"."""
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
sentinel = object()
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS:
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
return sentinel
orig, orig_delay = _patch_open_once(fake)
try:
handle = rp.RmHandle.open(0)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check("transient failure retried until success", handle is sentinel)
check(
"succeeds on the final allowed attempt",
attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_handle_open_reports_persistent_allocation_failure() -> None:
from nvcurve.hal import rm_power as rp
attempts: list[int] = []
@classmethod
def fake(klass, gpu_index: int):
attempts.append(gpu_index)
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
orig, orig_delay = _patch_open_once(fake)
try:
try:
rp.RmHandle.open(0)
check("persistent failure still raises", False)
except RmPowerError as exc:
check(
"persistent failure still raises with the original error",
"status 0x1f" in str(exc),
)
finally:
rp.RmHandle._open_once = orig
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
check(
"gives up after the bounded attempt count",
len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS,
)
def test_propagates_rm_query_failure() -> None: def test_propagates_rm_query_failure() -> None:
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0) pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
try: try:
@@ -342,6 +414,8 @@ def main() -> int:
test_resolves_pci_identity_when_minor_and_rm_orders_differ, test_resolves_pci_identity_when_minor_and_rm_orders_differ,
test_does_not_fall_back_to_another_gpu_when_pci_is_missing, test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
test_rejects_nonzero_pci_function, test_rejects_nonzero_pci_function,
test_handle_open_retries_transient_allocation_failure,
test_handle_open_reports_persistent_allocation_failure,
test_propagates_rm_query_failure, test_propagates_rm_query_failure,
] ]
for t in tests: for t in tests: