GSP transiently rejects NV0080 object allocation with status 0x1f when busy — reliably on the first probe in a fresh server process and intermittently right after RM operations (e.g. a power-limit write). The probe surfaced that transient rejection as rm_power_supported=false, making the Experimental Power Control block disappear in the UI until a reload re-probed successfully. - hal/rm_power: RmHandle.open() now retries the handle allocation (3 attempts, 100 ms apart) before reporting the interface unavailable; covers probe, set, and enable-check paths. Each attempt cleans up its own fds, so retries are safe and idempotent. - PerformancePanel: confirm a negative rm_power_supported reading with one follow-up fetch before rendering it, so a single flaky probe can't hide the block or flip it into "interface unavailable"; the confirm has its own error boundary so a failed second request keeps the first (valid) read. Also added a monotonic fetch-seq guard so a slow response for a previously selected GPU cannot clobber fresher state. - tests: cover transient-failure-retry and persistent-failure paths for RmHandle.open. Verified live: first probe after restart 5/5 (was 0/3), 150 rapid set+get cycles with 0 probe flips and 0 set failures.
430 lines
16 KiB
Python
430 lines
16 KiB
Python
"""Unit tests for the RM power-limit interface (fake RM, no hardware).
|
|
|
|
Standalone (no pytest required):
|
|
|
|
python tests/test_rm_power.py
|
|
|
|
Also works under pytest if available. Ports the test battery from LACT PR
|
|
#1205 (ilya-zlobintsev/LACT): layout discovery, NVML cross-validation,
|
|
write minimality, readback verification, and failure restoration.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
from nvcurve.hal.rm_power import ( # noqa: E402
|
|
_CTRL_GPU_GET_ATTACHED_IDS,
|
|
_CTRL_GPU_GET_ID_INFO_V2,
|
|
_CTRL_GPU_GET_PCI_INFO,
|
|
_PWR_GET_CONTROL,
|
|
_PWR_GET_INFO,
|
|
_PWR_SET_CONTROL,
|
|
EXTENDED_LAYOUT,
|
|
LEGACY_LAYOUT,
|
|
PciLocation,
|
|
PowerLimitBounds,
|
|
RmPowerError,
|
|
_u32,
|
|
probe,
|
|
resolve_gpu_instance,
|
|
set_limit,
|
|
)
|
|
|
|
PASS = 0
|
|
FAIL = 0
|
|
|
|
|
|
def check(name: str, cond: bool) -> None:
|
|
global PASS, FAIL
|
|
if cond:
|
|
PASS += 1
|
|
print(f" PASS {name}")
|
|
else:
|
|
FAIL += 1
|
|
print(f" FAIL {name}")
|
|
|
|
|
|
BOUNDS = PowerLimitBounds(min_mw=250_000, default_mw=300_000, max_mw=325_000)
|
|
|
|
|
|
class FakeRm:
|
|
"""In-memory fake of the RM power-limit client (both wire layouts)."""
|
|
|
|
def __init__(self, layout, current: int) -> None:
|
|
self.layout = layout
|
|
self.control = bytearray(layout.control_size)
|
|
self.control[0:8] = bytes([0xFF, 0, 0, 0, 1, 0, 0, 0])
|
|
self.control[layout.request_at - 4 : layout.request_at] = bytes(
|
|
[0x67, 0x67, 0, 0]
|
|
)
|
|
self.control[layout.request_at : layout.request_at + 4] = current.to_bytes(
|
|
4, "little"
|
|
)
|
|
self.control[layout.client_at] = 0xFE
|
|
self.reads: list[tuple[int, int]] = []
|
|
self.writes: list[bytes] = []
|
|
self.fail_first_write = False
|
|
self.fail_readback = False
|
|
self.fail_restore = False
|
|
|
|
def query(self, cmd: int, data: bytearray) -> None:
|
|
if cmd == _PWR_GET_INFO:
|
|
self.reads.append((cmd, len(data)))
|
|
if len(data) != self.layout.info_size:
|
|
raise RmPowerError("Unsupported INFO size")
|
|
data[0:8] = bytes([0xFF, 0, 0, 0, 1, 0, 0, 0])
|
|
for index, value in enumerate([250_000, 300_000, 325_000]):
|
|
offset = self.layout.info_min_at + 4 * index
|
|
data[offset : offset + 4] = value.to_bytes(4, "little")
|
|
elif cmd == _PWR_GET_CONTROL:
|
|
self.reads.append((cmd, len(data)))
|
|
if len(data) != self.layout.control_size:
|
|
raise RmPowerError("Unsupported CONTROL size")
|
|
if data[self.layout.client_at] != 0xFE:
|
|
raise AssertionError("unexpected client selector on GET")
|
|
if self.fail_readback and len(self.writes) == 1:
|
|
raise RmPowerError("readback unavailable")
|
|
data[:] = self.control
|
|
elif cmd == _PWR_SET_CONTROL:
|
|
if len(data) != self.layout.control_size:
|
|
raise AssertionError("bad SET size")
|
|
if data[4:8] != (1).to_bytes(4, "little"):
|
|
raise AssertionError("bad SET mask")
|
|
if data[self.layout.client_at] != 0xFE:
|
|
raise AssertionError("bad SET client selector")
|
|
# Only the request field may differ from the current state.
|
|
for i, (a, b) in enumerate(zip(data, self.control, strict=True)):
|
|
if self.layout.request_at <= i < self.layout.request_at + 4:
|
|
continue
|
|
if a != b:
|
|
raise AssertionError(f"SET modified byte {i:#x}")
|
|
self.writes.append(bytes(data))
|
|
if self.fail_restore and len(self.writes) > 1:
|
|
raise RmPowerError("restore unavailable")
|
|
self.control[:] = data
|
|
if self.fail_first_write and len(self.writes) == 1:
|
|
raise RmPowerError("SET failed after modifying hardware")
|
|
else:
|
|
raise AssertionError(f"unexpected command {cmd:#x}")
|
|
|
|
|
|
def test_detects_both_layouts_with_gets_without_a_driver_version() -> None:
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
rm = FakeRm(layout, 250_000)
|
|
support = probe(BOUNDS, 250_000, rm.query)
|
|
check(
|
|
f"{layout.name}: detected",
|
|
support.bounds == BOUNDS and support.layout == layout,
|
|
)
|
|
expected = (
|
|
[(_PWR_GET_INFO, 0x924), (_PWR_GET_CONTROL, 0x328)]
|
|
if layout == EXTENDED_LAYOUT
|
|
else [
|
|
(_PWR_GET_INFO, 0x924),
|
|
(_PWR_GET_INFO, 0x488),
|
|
(_PWR_GET_CONTROL, 0x188),
|
|
]
|
|
)
|
|
check(f"{layout.name}: GETs only, expected sequence", rm.reads == expected)
|
|
check(f"{layout.name}: no writes during discovery", rm.writes == [])
|
|
|
|
|
|
def test_unknown_layout_and_nvml_mismatches_never_write() -> None:
|
|
calls: list[tuple[int, int]] = []
|
|
|
|
def failing(cmd: int, data: bytearray) -> None:
|
|
calls.append((cmd, len(data)))
|
|
raise RmPowerError("Unsupported payload")
|
|
|
|
try:
|
|
probe(BOUNDS, 250_000, failing)
|
|
check("unknown layout rejected", False)
|
|
except RmPowerError:
|
|
check("unknown layout rejected", True)
|
|
check(
|
|
"unknown layout: only GETs attempted",
|
|
calls == [(_PWR_GET_INFO, 0x924), (_PWR_GET_INFO, 0x488)],
|
|
)
|
|
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
rm = FakeRm(layout, 250_000)
|
|
try:
|
|
probe(BOUNDS, 300_000, rm.query)
|
|
check(f"{layout.name}: current mismatch rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: current mismatch rejected", True)
|
|
other_bounds = PowerLimitBounds(
|
|
min_mw=BOUNDS.min_mw, default_mw=BOUNDS.default_mw, max_mw=350_000
|
|
)
|
|
try:
|
|
probe(other_bounds, 250_000, rm.query)
|
|
check(f"{layout.name}: bounds mismatch rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: bounds mismatch rejected", True)
|
|
check(f"{layout.name}: no writes on mismatch", rm.writes == [])
|
|
|
|
|
|
def test_rejects_unrecognized_headers_masks_and_client_values() -> None:
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
for at, value in [(0, 0), (4, 3), (layout.client_at, 0xF8)]:
|
|
rm = FakeRm(layout, 250_000)
|
|
rm.control[at] = value
|
|
try:
|
|
probe(BOUNDS, 250_000, rm.query)
|
|
check(f"{layout.name}: bad header/client rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: bad header/client rejected", True)
|
|
check(f"{layout.name}: no writes on bad header", rm.writes == [])
|
|
for current in (0, 0xFFFFFFFF):
|
|
rm = FakeRm(layout, current)
|
|
try:
|
|
probe(BOUNDS, current, rm.query)
|
|
check(f"{layout.name}: empty request rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: empty request rejected", True)
|
|
# The extended layout has additional mask words. Accepting only its low
|
|
# word would allow an unexpected client to be included in a later SET.
|
|
rm = FakeRm(EXTENDED_LAYOUT, 250_000)
|
|
rm.control[8] = 1
|
|
try:
|
|
probe(BOUNDS, 250_000, rm.query)
|
|
check("extended: nonzero mask word rejected", False)
|
|
except RmPowerError:
|
|
check("extended: nonzero mask word rejected", True)
|
|
check("extended: no writes on mask violation", rm.writes == [])
|
|
|
|
|
|
def test_changes_only_fe_request_and_keeps_vbios_maximum() -> None:
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
rm = FakeRm(layout, 250_000)
|
|
support = probe(BOUNDS, 250_000, rm.query)
|
|
for cap in (150_000, 30_000, 250_000):
|
|
set_limit(cap, support, rm.query)
|
|
check(
|
|
f"{layout.name}: set {cap} mW",
|
|
_u32(rm.control, layout.request_at) == cap,
|
|
)
|
|
writes = len(rm.writes)
|
|
for cap in (0, 29_999, 325_001, 350_000, 0xFFFFFFFF):
|
|
try:
|
|
set_limit(cap, support, rm.query)
|
|
check(f"{layout.name}: out-of-range {cap} rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: out-of-range {cap} rejected", True)
|
|
check(
|
|
f"{layout.name}: no writes for out-of-range caps",
|
|
len(rm.writes) == writes,
|
|
)
|
|
|
|
|
|
def test_restores_previous_below_minimum_request_after_set_or_readback_failure() -> (
|
|
None
|
|
):
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
for fail_set in (False, True):
|
|
rm = FakeRm(layout, 100_000)
|
|
original = bytes(rm.control)
|
|
rm.fail_first_write = fail_set
|
|
rm.fail_readback = not fail_set
|
|
support = probe(BOUNDS, 100_000, rm.query)
|
|
try:
|
|
set_limit(150_000, support, rm.query)
|
|
check(f"{layout.name}: failure reported", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: failure reported", True)
|
|
check(f"{layout.name}: restore issued", len(rm.writes) == 2)
|
|
check(
|
|
f"{layout.name}: previous request restored",
|
|
bytes(rm.control) == original,
|
|
)
|
|
|
|
|
|
def test_reports_restore_failure_and_rejects_wrong_client_before_writing() -> None:
|
|
for layout in (EXTENDED_LAYOUT, LEGACY_LAYOUT):
|
|
rm = FakeRm(layout, 100_000)
|
|
rm.fail_first_write = True
|
|
rm.fail_restore = True
|
|
support = probe(BOUNDS, 100_000, rm.query)
|
|
try:
|
|
set_limit(150_000, support, rm.query)
|
|
check(f"{layout.name}: restore failure reported", False)
|
|
except RmPowerError as exc:
|
|
check(
|
|
f"{layout.name}: restore failure reported",
|
|
"restoration also failed" in str(exc),
|
|
)
|
|
rm = FakeRm(layout, 250_000)
|
|
rm.control[layout.client_at] = 0xF8
|
|
try:
|
|
set_limit(150_000, support, rm.query)
|
|
check(f"{layout.name}: wrong client rejected", False)
|
|
except RmPowerError:
|
|
check(f"{layout.name}: wrong client rejected", True)
|
|
check(f"{layout.name}: no writes for wrong client", rm.writes == [])
|
|
|
|
|
|
# ── PCI identity → RM instance resolution ────────────────────────────────────
|
|
|
|
|
|
def test_resolves_pci_identity_when_minor_and_rm_orders_differ() -> None:
|
|
# This host has Ada at minor 5/RM 4 and the 5090 at minor 4/RM 5.
|
|
# IDs are opaque and enumeration order must not select the device.
|
|
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
|
|
instances = resolve_gpu_instance(pci, lambda cmd, data: _fake_root(cmd, data))
|
|
check("resolves by PCI identity", instances == (5, 2))
|
|
|
|
|
|
def _fake_root(cmd: int, data: bytearray) -> None:
|
|
if cmd == _CTRL_GPU_GET_ATTACHED_IDS:
|
|
data[0:4] = (0x2E00).to_bytes(4, "little")
|
|
data[4:8] = (0x0D00).to_bytes(4, "little")
|
|
elif cmd == _CTRL_GPU_GET_PCI_INFO:
|
|
gpu_id = _u32(data, 0)
|
|
bus = 0x2E if gpu_id == 0x2E00 else 0x0D
|
|
data[8:10] = bus.to_bytes(2, "little")
|
|
elif cmd == _CTRL_GPU_GET_ID_INFO_V2:
|
|
if _u32(data, 0) != 0x0D00:
|
|
raise AssertionError("unexpected gpu id in ID_INFO_V2")
|
|
data[8:12] = (5).to_bytes(4, "little")
|
|
data[12:16] = (2).to_bytes(4, "little")
|
|
else:
|
|
raise AssertionError(f"unexpected command {cmd:#x}")
|
|
|
|
|
|
def test_does_not_fall_back_to_another_gpu_when_pci_is_missing() -> None:
|
|
pci = PciLocation(domain=1, bus=0x0D, dev=0, func=0)
|
|
|
|
def query(cmd: int, data: bytearray) -> None:
|
|
if cmd == _CTRL_GPU_GET_ATTACHED_IDS:
|
|
data[0:4] = (0x0D00).to_bytes(4, "little")
|
|
elif cmd == _CTRL_GPU_GET_PCI_INFO:
|
|
data[8:10] = (0x0D).to_bytes(2, "little")
|
|
else:
|
|
raise AssertionError("must not allocate a GPU from another PCI domain")
|
|
|
|
try:
|
|
resolve_gpu_instance(pci, query)
|
|
check("foreign PCI domain rejected", False)
|
|
except RmPowerError:
|
|
check("foreign PCI domain rejected", True)
|
|
|
|
|
|
def test_rejects_nonzero_pci_function() -> None:
|
|
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=1)
|
|
try:
|
|
resolve_gpu_instance(pci, lambda cmd, data: None)
|
|
check("nonzero function rejected", False)
|
|
except RmPowerError:
|
|
check("nonzero function rejected", True)
|
|
|
|
|
|
# ── Transient RM allocation retries (RmHandle.open) ─────────────────────────
|
|
|
|
|
|
def _patch_open_once(fake) -> tuple:
|
|
from nvcurve.hal import rm_power as _rp
|
|
|
|
orig = _rp.RmHandle._open_once
|
|
orig_delay = _rp.RmHandle._ALLOCATION_RETRY_DELAY_S
|
|
_rp.RmHandle._open_once = fake
|
|
_rp.RmHandle._ALLOCATION_RETRY_DELAY_S = 0
|
|
return orig, orig_delay
|
|
|
|
|
|
def test_handle_open_retries_transient_allocation_failure() -> None:
|
|
"""GSP rejects RM object allocation with status 0x1f when busy (first
|
|
probe in a fresh process, or a request right after a power-limit write);
|
|
it succeeds on immediate re-attempt. A single transient rejection must
|
|
not surface as "interface not supported"."""
|
|
from nvcurve.hal import rm_power as rp
|
|
|
|
attempts: list[int] = []
|
|
sentinel = object()
|
|
|
|
@classmethod
|
|
def fake(klass, gpu_index: int):
|
|
attempts.append(gpu_index)
|
|
if len(attempts) < rp.RmHandle._ALLOCATION_ATTEMPTS:
|
|
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
|
|
return sentinel
|
|
|
|
orig, orig_delay = _patch_open_once(fake)
|
|
try:
|
|
handle = rp.RmHandle.open(0)
|
|
finally:
|
|
rp.RmHandle._open_once = orig
|
|
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
|
|
check("transient failure retried until success", handle is sentinel)
|
|
check(
|
|
"succeeds on the final allowed attempt",
|
|
attempts == [0] * rp.RmHandle._ALLOCATION_ATTEMPTS,
|
|
)
|
|
|
|
|
|
def test_handle_open_reports_persistent_allocation_failure() -> None:
|
|
from nvcurve.hal import rm_power as rp
|
|
|
|
attempts: list[int] = []
|
|
|
|
@classmethod
|
|
def fake(klass, gpu_index: int):
|
|
attempts.append(gpu_index)
|
|
raise RmPowerError("RM class 0x80 allocation failed (status 0x1f)")
|
|
|
|
orig, orig_delay = _patch_open_once(fake)
|
|
try:
|
|
try:
|
|
rp.RmHandle.open(0)
|
|
check("persistent failure still raises", False)
|
|
except RmPowerError as exc:
|
|
check(
|
|
"persistent failure still raises with the original error",
|
|
"status 0x1f" in str(exc),
|
|
)
|
|
finally:
|
|
rp.RmHandle._open_once = orig
|
|
rp.RmHandle._ALLOCATION_RETRY_DELAY_S = orig_delay
|
|
check(
|
|
"gives up after the bounded attempt count",
|
|
len(attempts) == rp.RmHandle._ALLOCATION_ATTEMPTS,
|
|
)
|
|
|
|
|
|
def test_propagates_rm_query_failure() -> None:
|
|
pci = PciLocation(domain=0, bus=0x0D, dev=0, func=0)
|
|
try:
|
|
resolve_gpu_instance(
|
|
pci, lambda cmd, data: (_ for _ in ()).throw(RmPowerError("RM unavailable"))
|
|
)
|
|
check("RM query failure propagated", False)
|
|
except RmPowerError as exc:
|
|
check("RM query failure propagated", "RM unavailable" in str(exc))
|
|
|
|
|
|
def main() -> int:
|
|
tests = [
|
|
test_detects_both_layouts_with_gets_without_a_driver_version,
|
|
test_unknown_layout_and_nvml_mismatches_never_write,
|
|
test_rejects_unrecognized_headers_masks_and_client_values,
|
|
test_changes_only_fe_request_and_keeps_vbios_maximum,
|
|
test_restores_previous_below_minimum_request_after_set_or_readback_failure,
|
|
test_reports_restore_failure_and_rejects_wrong_client_before_writing,
|
|
test_resolves_pci_identity_when_minor_and_rm_orders_differ,
|
|
test_does_not_fall_back_to_another_gpu_when_pci_is_missing,
|
|
test_rejects_nonzero_pci_function,
|
|
test_handle_open_retries_transient_allocation_failure,
|
|
test_handle_open_reports_persistent_allocation_failure,
|
|
test_propagates_rm_query_failure,
|
|
]
|
|
for t in tests:
|
|
print(f"== {t.__name__} ==")
|
|
t()
|
|
print(f"\n{PASS} passed, {FAIL} failed")
|
|
return 1 if FAIL else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|