fix: retry transient RM allocation failures so power control block doesn't vanish
GSP transiently rejects NV0080 object allocation with status 0x1f when busy — reliably on the first probe in a fresh server process and intermittently right after RM operations (e.g. a power-limit write). The probe surfaced that transient rejection as rm_power_supported=false, making the Experimental Power Control block disappear in the UI until a reload re-probed successfully. - hal/rm_power: RmHandle.open() now retries the handle allocation (3 attempts, 100 ms apart) before reporting the interface unavailable; covers probe, set, and enable-check paths. Each attempt cleans up its own fds, so retries are safe and idempotent. - PerformancePanel: confirm a negative rm_power_supported reading with one follow-up fetch before rendering it, so a single flaky probe can't hide the block or flip it into "interface unavailable"; the confirm has its own error boundary so a failed second request keeps the first (valid) read. Also added a monotonic fetch-seq guard so a slow response for a previously selected GPU cannot clobber fresher state. - tests: cover transient-failure-retry and persistent-failure paths for RmHandle.open. Verified live: first probe after restart 5/5 (was 0/3), 150 rapid set+get cycles with 0 probe flips and 0 set failures.
This commit is contained in:
1 parent
6afdd15d2a
commit
2e8e580d79
3 files changed
+135
-2
No files matched your search
@@ -1,4 +1,4 @@
|
|||||||
import { useState, useEffect } from "react";
|
import { useState, useEffect, useRef } from "react";
|
||||||
import { Check, X, RotateCcw } from "lucide-react";
|
import { Check, X, RotateCcw } from "lucide-react";
|
||||||
import { api } from "../../api/client.js";
|
import { api } from "../../api/client.js";
|
||||||
import { useCurveStore } from "../../store/curveStore.js";
|
import { useCurveStore } from "../../store/curveStore.js";
|
||||||
@@ -17,17 +17,47 @@ export function PerformancePanel() {
|
|||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [confirmApply, setConfirmApply] = useState(false);
|
const [confirmApply, setConfirmApply] = useState(false);
|
||||||
const [confirmReset, setConfirmReset] = useState(false);
|
const [confirmReset, setConfirmReset] = useState(false);
|
||||||
|
// Monotonic fetch counter: a response may only commit state (or toast /
|
||||||
|
// clear loading) if it is still the latest fetch for this panel, so a slow
|
||||||
|
// response for a previously selected GPU cannot clobber fresher results.
|
||||||
|
const fetchSeq = useRef(0);
|
||||||
|
|
||||||
async function fetchLimits() {
|
async function fetchLimits() {
|
||||||
|
const seq = ++fetchSeq.current;
|
||||||
try {
|
try {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
setLimits(await api.limits(selectedGpuIndex));
|
let next = await api.limits(selectedGpuIndex);
|
||||||
|
// The RM support probe can flake transiently (the driver briefly
|
||||||
|
// rejects RM object allocation when busy) and report
|
||||||
|
// rm_power_supported=false for one read. A single failed read must not
|
||||||
|
// make the Experimental Power Control block vanish — or flip it into
|
||||||
|
// "interface unavailable" — so confirm a negative reading with one
|
||||||
|
// follow-up fetch before trusting it. The confirm has its own error
|
||||||
|
// boundary: if it throws (network blip), keep the first result, which
|
||||||
|
// is still a valid read, rather than discarding it into the outer catch.
|
||||||
|
if (!next.rm_power_supported) {
|
||||||
|
try {
|
||||||
|
const again = await api.limits(selectedGpuIndex);
|
||||||
|
if (again.rm_power_supported) {
|
||||||
|
next = again;
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
|
// keep first result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (seq === fetchSeq.current) {
|
||||||
|
setLimits(next);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
if (seq === fetchSeq.current) {
|
||||||
toast.error("Failed to load performance limits");
|
toast.error("Failed to load performance limits");
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
if (seq === fetchSeq.current) {
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Data fetch on GPU change — setState calls happen after the await, not
|
// Data fetch on GPU change — setState calls happen after the await, not
|
||||||
// synchronously in the effect body (rule false-positive on async fetch).
|
// synchronously in the effect body (rule false-positive on async fetch).
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import struct
|
import struct
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@@ -343,8 +344,36 @@ class RmHandle:
|
|||||||
self.device_handle = device_handle
|
self.device_handle = device_handle
|
||||||
self.subdevice_handle = subdevice_handle
|
self.subdevice_handle = subdevice_handle
|
||||||
|
|
||||||
|
# Transient RM allocation failures (observed: NV0080 alloc rejected with
|
||||||
|
# status 0x1f) happen when GSP is still busy with a preceding operation —
|
||||||
|
# e.g. the first probe in a fresh process, or a request issued directly
|
||||||
|
# after a power-limit write. The allocation succeeds on immediate
|
||||||
|
# re-attempt, so retry a short, bounded number of times before reporting
|
||||||
|
# the interface as unavailable.
|
||||||
|
_ALLOCATION_ATTEMPTS = 3
|
||||||
|
_ALLOCATION_RETRY_DELAY_S = 0.1
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def open(cls, gpu_index: int) -> RmHandle:
|
def open(cls, gpu_index: int) -> RmHandle:
|
||||||
|
"""Open an RM handle, retrying transient driver allocation failures."""
|
||||||
|
last_error: RmPowerError | None = None
|
||||||
|
for attempt in range(cls._ALLOCATION_ATTEMPTS):
|
||||||
|
try:
|
||||||
|
return cls._open_once(gpu_index)
|
||||||
|
except RmPowerError as exc:
|
||||||
|
last_error = exc
|
||||||
|
if attempt + 1 < cls._ALLOCATION_ATTEMPTS:
|
||||||
|
log.debug(
|
||||||
|
"RM handle open failed (attempt %d/%d): %s",
|
||||||
|
attempt + 1,
|
||||||
|
cls._ALLOCATION_ATTEMPTS,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
time.sleep(cls._ALLOCATION_RETRY_DELAY_S)
|
||||||
|
raise last_error or RmPowerError("RM handle open failed")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _open_once(cls, gpu_index: int) -> RmHandle:
|
||||||
"""Open an RM handle for the GPU at the given NVML index.
|
"""Open an RM handle for the GPU at the given NVML index.
|
||||||
|
|
||||||
The RM device/subdevice instances are resolved by PCI identity
|
The RM device/subdevice instances are resolved by PCI identity
|
||||||
|
|||||||
@@ -320,6 +320,78 @@ def test_rejects_nonzero_pci_function() -> None:
|
|||||||
check("nonzero function rejected", True)
|
check("nonzero function rejected", True)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Transient RM allocation retries (RmHandle.open) ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_open_once(fake) -> tuple:
|
||||||
|
from nvcurve.hal import rm_power as _rp
|
||||||
|
|
||||||
|
orig = _rp.RmHandle._open_once
|
||||||
|
orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S
|
||||||
|
_rp.RmHandle._open_once = fake
|
||||||
|
_rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0
|
||||||
|
return orig, orig_delay
|
||||||
|
|
||||||
|
|
||||||
|
def test_handle_open_retries_transient_allocation_failure() -> None:
|
||||||
|
"""GSP rejects RM object allocation with status 0x1f when busy (first
|
||||||
|
probe in a fresh process, or a request right after a power-limit write);
|
||||||
|
it succeeds on immediate re-attempt. A single transient rejection must
|
||||||
|
not surface as "interface not supported"."""
|
||||||
|
from nvcurve.hal import rm_power as rp
|
||||||
|
|
||||||
|
attempts: list[int] = []
|
||||||
|
sentinel = object()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fake(klass, gpu_index: int):
|
||||||
|
attempts.append(gpu_index)
|
||||||
|
if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS:
|
||||||
|
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
|
||||||
|
return sentinel
|
||||||
|
|
||||||
|
orig, orig_delay = _patch_open_once(fake)
|
||||||
|
try:
|
||||||
|
handle = rp.RmHandle.open(0)
|
||||||
|
finally:
|
||||||
|
rp.RmHandle._open_once = orig
|
||||||
|
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
|
||||||
|
check("transient failure retried until success", handle is sentinel)
|
||||||
|
check(
|
||||||
|
"succeeds on the final allowed attempt",
|
||||||
|
attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_handle_open_reports_persistent_allocation_failure() -> None:
|
||||||
|
from nvcurve.hal import rm_power as rp
|
||||||
|
|
||||||
|
attempts: list[int] = []
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def fake(klass, gpu_index: int):
|
||||||
|
attempts.append(gpu_index)
|
||||||
|
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
|
||||||
|
|
||||||
|
orig, orig_delay = _patch_open_once(fake)
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
rp.RmHandle.open(0)
|
||||||
|
check("persistent failure still raises", False)
|
||||||
|
except RmPowerError as exc:
|
||||||
|
check(
|
||||||
|
"persistent failure still raises with the original error",
|
||||||
|
"status 0x1f" in str(exc),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
rp.RmHandle._open_once = orig
|
||||||
|
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
|
||||||
|
check(
|
||||||
|
"gives up after the bounded attempt count",
|
||||||
|
len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_propagates_rm_query_failure() -> None:
|
def test_propagates_rm_query_failure() -> None:
|
||||||
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
|
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
|
||||||
try:
|
try:
|
||||||
@@ -342,6 +414,8 @@ def main() -> int:
|
|||||||
test_resolves_pci_identity_when_minor_and_rm_orders_differ,
|
test_resolves_pci_identity_when_minor_and_rm_orders_differ,
|
||||||
test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
|
test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
|
||||||
test_rejects_nonzero_pci_function,
|
test_rejects_nonzero_pci_function,
|
||||||
|
test_handle_open_retries_transient_allocation_failure,
|
||||||
|
test_handle_open_reports_persistent_allocation_failure,
|
||||||
test_propagates_rm_query_failure,
|
test_propagates_rm_query_failure,
|
||||||
]
|
]
|
||||||
for t in tests:
|
for t in tests:
|
||||||
|
|||||||
Reference in new issue
Block a user