|
|
|
@@ -8,6 +8,11 @@ Clock offsets use nvmlDeviceSetClockOffsets / nvmlDeviceGetClockOffsets
|
|
|
|
|
as a fallback when the new API is unavailable or returns an error.
|
|
|
|
|
set_clock_offsets accepts Optional values and only touches the domains
|
|
|
|
|
that are explicitly specified, leaving others unchanged on hardware.
|
|
|
|
|
|
|
|
|
|
All public functions in this module use the *reported* clock domain for the
|
|
|
|
|
memory offset — the domain of NVML_CLOCK_MEM, i.e. what nvidia-smi shows.
|
|
|
|
|
The conversion to the driver's internal domain happens here (see
|
|
|
|
|
_MEM_OFFSET_DOMAIN_FACTOR).
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
import ctypes
|
|
|
|
@@ -179,6 +184,23 @@ _CLOCK_OFFSET_VER = (1 << 24) | ctypes.sizeof(_ClockOffset) # = 0x01000010 (16
|
|
|
|
|
_NVML_CLOCK_GRAPHICS = 0
|
|
|
|
|
_NVML_CLOCK_MEM = 2
|
|
|
|
|
|
|
|
|
|
# Domain factor for the memory-V/F offset APIs (nvmlDeviceSetMemClkVfOffset /
|
|
|
|
|
# nvmlDeviceSetClockOffsets(MEM) and their readback + range queries).
|
|
|
|
|
#
|
|
|
|
|
# Measured on RTX 5090, driver 615.78.08 (GDDR7/Blackwell): the driver takes
|
|
|
|
|
# and reports the offset in a domain that is 2x the memory clock NVML
|
|
|
|
|
# actually reports (NVML_CLOCK_MEM, what nvidia-smi shows). Applying +3000
|
|
|
|
|
# moves the reported clock by only +1500 MHz; +6000 is the "real" +3000 in
|
|
|
|
|
# nvidia-smi terms. (Cross-check: NvAPI GetClockBoostRanges says -1000/+3000
|
|
|
|
|
# while nvmlDeviceGetMemClkMinMaxVfOffset says -2000/+6000 — the same range
|
|
|
|
|
# in the two domains.)
|
|
|
|
|
#
|
|
|
|
|
# This module therefore converts at the NVML boundary:
|
|
|
|
|
# set: nvml_value = reported_offset * _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
# get: reported_offset = nvml_value / _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
# range: reported = nvml_range / _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
_MEM_OFFSET_DOMAIN_FACTOR = 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _make_clock_offset(
|
|
|
|
|
clock_type: int, pstate: int = 0, offset_mhz: int = 0
|
|
|
|
@@ -208,10 +230,31 @@ def _try_nvml_fn(name: str):
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _nvml_raw_call(name: str, handle, info) -> int | None:
|
|
|
|
|
"""Call an NVML function via its raw function pointer; return the real driver rc.
|
|
|
|
|
|
|
|
|
|
nvidia-ml-py's nvmlDeviceGet/SetClockOffsets wrappers discard the driver's
|
|
|
|
|
return code and always report success (verified in 13.615.71): on a driver
|
|
|
|
|
where the API errors, the zero-filled struct would be treated as valid and
|
|
|
|
|
the deprecated-API fallbacks would never run. Returns None when the raw
|
|
|
|
|
pointer is unavailable or cannot be called.
|
|
|
|
|
"""
|
|
|
|
|
try:
|
|
|
|
|
get_ptr = getattr(pynvml, "_nvmlGetFunctionPointer", None)
|
|
|
|
|
if get_ptr is None:
|
|
|
|
|
return None
|
|
|
|
|
return int(get_ptr(name)(handle, ctypes.byref(info)))
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
log.debug("_nvml_raw_call(%s): %s", name, exc)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_clock_offsets(gpu_index: int = 0) -> dict:
|
|
|
|
|
"""Return GPC and memory clock offsets (MHz).
|
|
|
|
|
|
|
|
|
|
Keys: gpc_offset_mhz, mem_offset_mhz (both int or None on failure).
|
|
|
|
|
mem_offset_mhz is in the reported-clock domain (what nvidia-smi shows);
|
|
|
|
|
the driver's 2x-internal value is converted back here.
|
|
|
|
|
Calls nvmlDeviceGetClockOffsets once per clock domain (GRAPHICS, MEM).
|
|
|
|
|
"""
|
|
|
|
|
out: dict[str, int | None] = {"gpc_offset_mhz": None, "mem_offset_mhz": None}
|
|
|
|
@@ -220,12 +263,14 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
|
|
|
|
|
try:
|
|
|
|
|
handle = _get_handle(gpu_index)
|
|
|
|
|
|
|
|
|
|
# Try pynvml wrapper first (nvidia-ml-py ≥ 12 exposes it correctly).
|
|
|
|
|
# Fall back to ctypes-direct if pynvml doesn't have it.
|
|
|
|
|
# Prefer the raw function pointer: nvidia-ml-py's
|
|
|
|
|
# nvmlDeviceGetClockOffsets wrapper discards the driver's return code
|
|
|
|
|
# and always reports success (verified in 13.615.71), so an erroring
|
|
|
|
|
# driver would yield a zeroed struct we'd treat as valid and the
|
|
|
|
|
# deprecated fallback below would never run. The pynvml wrapper and
|
|
|
|
|
# the CDLL lookup remain as last resorts.
|
|
|
|
|
_pynvml_get = getattr(pynvml, "nvmlDeviceGetClockOffsets", None)
|
|
|
|
|
fn_get = (
|
|
|
|
|
_try_nvml_fn("nvmlDeviceGetClockOffsets") if _pynvml_get is None else None
|
|
|
|
|
)
|
|
|
|
|
fn_get = _try_nvml_fn("nvmlDeviceGetClockOffsets")
|
|
|
|
|
|
|
|
|
|
used_new_api = False
|
|
|
|
|
for clock_type, key in (
|
|
|
|
@@ -234,6 +279,8 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
|
|
|
|
|
):
|
|
|
|
|
info = _make_clock_offset(clock_type, pstate=0)
|
|
|
|
|
try:
|
|
|
|
|
rc = _nvml_raw_call("nvmlDeviceGetClockOffsets", handle, info)
|
|
|
|
|
if rc is None:
|
|
|
|
|
if _pynvml_get is not None:
|
|
|
|
|
rc = _pynvml_get(handle, ctypes.byref(info))
|
|
|
|
|
elif fn_get is not None:
|
|
|
|
@@ -241,7 +288,10 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
|
|
|
|
|
else:
|
|
|
|
|
break
|
|
|
|
|
if rc == 0:
|
|
|
|
|
out[key] = int(info.clockOffsetMHz)
|
|
|
|
|
val = int(info.clockOffsetMHz)
|
|
|
|
|
if key == "mem_offset_mhz":
|
|
|
|
|
val = int(val / _MEM_OFFSET_DOMAIN_FACTOR)
|
|
|
|
|
out[key] = val
|
|
|
|
|
used_new_api = True
|
|
|
|
|
else:
|
|
|
|
|
log.debug(
|
|
|
|
@@ -262,9 +312,8 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
|
|
|
|
|
if hasattr(pynvml, "nvmlDeviceGetMemClkVfOffset"):
|
|
|
|
|
try:
|
|
|
|
|
res = pynvml.nvmlDeviceGetMemClkVfOffset(handle)
|
|
|
|
|
out["mem_offset_mhz"] = int(
|
|
|
|
|
res[0] if isinstance(res, (list, tuple)) else res
|
|
|
|
|
)
|
|
|
|
|
raw = res[0] if isinstance(res, (list, tuple)) else res
|
|
|
|
|
out["mem_offset_mhz"] = int(raw / _MEM_OFFSET_DOMAIN_FACTOR)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
log.debug("nvmlDeviceGetMemClkVfOffset: %s", exc)
|
|
|
|
|
|
|
|
|
@@ -281,42 +330,52 @@ def set_clock_offsets(
|
|
|
|
|
"""Set clock offsets (MHz) for the specified domains only.
|
|
|
|
|
|
|
|
|
|
Pass None for a domain to leave it untouched on hardware.
|
|
|
|
|
mem_offset_mhz is in the reported-clock domain (what nvidia-smi shows);
|
|
|
|
|
it is multiplied by _MEM_OFFSET_DOMAIN_FACTOR before reaching the driver.
|
|
|
|
|
Calls nvmlDeviceSetClockOffsets once per requested domain (GRAPHICS, MEM).
|
|
|
|
|
Falls back to deprecated per-domain functions when the new API returns an
|
|
|
|
|
error (e.g. NVML_ERROR_DEPRECATED=25 on Blackwell with driver 590.x).
|
|
|
|
|
error (e.g. NVML_ERROR_DEPRECATED=26 on Blackwell with driver 590.x).
|
|
|
|
|
"""
|
|
|
|
|
if gpc_offset_mhz is None and mem_offset_mhz is None:
|
|
|
|
|
return True, "OK"
|
|
|
|
|
if not _NVML_AVAILABLE:
|
|
|
|
|
return False, "NVML not available (install nvidia-ml-py)"
|
|
|
|
|
# Convert to the driver's internal domain (2x the reported clock on
|
|
|
|
|
# GDDR7/Blackwell; see _MEM_OFFSET_DOMAIN_FACTOR).
|
|
|
|
|
mem_api = (
|
|
|
|
|
mem_offset_mhz * _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
if mem_offset_mhz is not None
|
|
|
|
|
else None
|
|
|
|
|
)
|
|
|
|
|
try:
|
|
|
|
|
handle = _get_handle(gpu_index)
|
|
|
|
|
|
|
|
|
|
domains = []
|
|
|
|
|
if gpc_offset_mhz is not None:
|
|
|
|
|
domains.append((_NVML_CLOCK_GRAPHICS, gpc_offset_mhz))
|
|
|
|
|
if mem_offset_mhz is not None:
|
|
|
|
|
domains.append((_NVML_CLOCK_MEM, mem_offset_mhz))
|
|
|
|
|
if mem_api is not None:
|
|
|
|
|
domains.append((_NVML_CLOCK_MEM, mem_api))
|
|
|
|
|
|
|
|
|
|
_pynvml_set = getattr(pynvml, "nvmlDeviceSetClockOffsets", None)
|
|
|
|
|
fn_set = (
|
|
|
|
|
_try_nvml_fn("nvmlDeviceSetClockOffsets") if _pynvml_set is None else None
|
|
|
|
|
)
|
|
|
|
|
fn_set = _try_nvml_fn("nvmlDeviceSetClockOffsets")
|
|
|
|
|
|
|
|
|
|
if _pynvml_set is not None or fn_set is not None:
|
|
|
|
|
# Same rationale as get_clock_offsets: the pynvml wrapper swallows the
|
|
|
|
|
# driver rc (always returns success), so go raw first. If every binding
|
|
|
|
|
# is missing, all_ok flips to False and we fall through to the
|
|
|
|
|
# deprecated path below.
|
|
|
|
|
all_ok = True
|
|
|
|
|
for clock_type, offset in domains:
|
|
|
|
|
info = _make_clock_offset(clock_type, pstate=0, offset_mhz=offset)
|
|
|
|
|
try:
|
|
|
|
|
rc = _nvml_raw_call("nvmlDeviceSetClockOffsets", handle, info)
|
|
|
|
|
if rc is None:
|
|
|
|
|
if _pynvml_set is not None:
|
|
|
|
|
rc = _pynvml_set(handle, ctypes.byref(info))
|
|
|
|
|
elif fn_set is not None:
|
|
|
|
|
rc = fn_set(handle, ctypes.byref(info))
|
|
|
|
|
else:
|
|
|
|
|
break
|
|
|
|
|
if rc != 0:
|
|
|
|
|
if rc is None or rc != 0:
|
|
|
|
|
log.debug(
|
|
|
|
|
"nvmlDeviceSetClockOffsets(type=%d) returned %d — trying fallback",
|
|
|
|
|
"nvmlDeviceSetClockOffsets(type=%d) unavailable or returned %r — trying fallback",
|
|
|
|
|
clock_type,
|
|
|
|
|
rc,
|
|
|
|
|
)
|
|
|
|
@@ -332,7 +391,8 @@ def set_clock_offsets(
|
|
|
|
|
break
|
|
|
|
|
if all_ok:
|
|
|
|
|
return True, "OK"
|
|
|
|
|
# Non-zero rc (e.g. 25=DEPRECATED on Blackwell) — fall through to deprecated path.
|
|
|
|
|
# Non-zero rc (e.g. 26=DEPRECATED on Blackwell), or the new API is
|
|
|
|
|
# unavailable — fall through to the deprecated path.
|
|
|
|
|
|
|
|
|
|
# Deprecated per-domain fallback (works on Blackwell/driver 590.x).
|
|
|
|
|
errs = []
|
|
|
|
@@ -343,11 +403,11 @@ def set_clock_offsets(
|
|
|
|
|
pynvml.nvmlDeviceSetGpcClkVfOffset(handle, gpc_offset_mhz)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
errs.append(f"GPC: {exc}")
|
|
|
|
|
if mem_offset_mhz is not None and hasattr(
|
|
|
|
|
if mem_api is not None and hasattr(
|
|
|
|
|
pynvml, "nvmlDeviceSetMemClkVfOffset"
|
|
|
|
|
):
|
|
|
|
|
try:
|
|
|
|
|
pynvml.nvmlDeviceSetMemClkVfOffset(handle, mem_offset_mhz)
|
|
|
|
|
pynvml.nvmlDeviceSetMemClkVfOffset(handle, mem_api)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
errs.append(f"MEM: {exc}")
|
|
|
|
|
if errs:
|
|
|
|
@@ -366,10 +426,14 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
|
|
|
|
|
"""Return the min/max allowed memory clock offset (MHz).
|
|
|
|
|
|
|
|
|
|
Keys: min_mem_offset_mhz, max_mem_offset_mhz.
|
|
|
|
|
Values are in the reported-clock domain (what nvidia-smi shows); the
|
|
|
|
|
driver's 2x-internal range is converted down here.
|
|
|
|
|
Uses nvmlDeviceGetMemClkMinMaxVfOffset; falls back to observed RTX values.
|
|
|
|
|
"""
|
|
|
|
|
# Observed RTX 5090 defaults (NvAPI GetClockBoostRanges says -1000/+3000).
|
|
|
|
|
out: dict[str, int] = {"min_mem_offset_mhz": -2000, "max_mem_offset_mhz": 3000}
|
|
|
|
|
# Observed RTX 5090 range in the reported domain
|
|
|
|
|
# (NvAPI GetClockBoostRanges: -1000/+3000; the NVML range call returns the
|
|
|
|
|
# same as -2000/+6000 in its internal domain).
|
|
|
|
|
out: dict[str, int] = {"min_mem_offset_mhz": -1000, "max_mem_offset_mhz": 3000}
|
|
|
|
|
if not _NVML_AVAILABLE:
|
|
|
|
|
return out
|
|
|
|
|
try:
|
|
|
|
@@ -378,15 +442,14 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
|
|
|
|
|
if hasattr(pynvml, "nvmlDeviceGetMemClkMinMaxVfOffset"):
|
|
|
|
|
result = pynvml.nvmlDeviceGetMemClkMinMaxVfOffset(handle)
|
|
|
|
|
if isinstance(result, (list, tuple)) and len(result) >= 2:
|
|
|
|
|
out["min_mem_offset_mhz"] = int(result[0])
|
|
|
|
|
out["max_mem_offset_mhz"] = int(result[1])
|
|
|
|
|
min_v, max_v = int(result[0]), int(result[1])
|
|
|
|
|
else:
|
|
|
|
|
min_v = getattr(result, "minOffset", None)
|
|
|
|
|
max_v = getattr(result, "maxOffset", None)
|
|
|
|
|
if min_v is not None:
|
|
|
|
|
out["min_mem_offset_mhz"] = int(min_v)
|
|
|
|
|
out["min_mem_offset_mhz"] = int(min_v / _MEM_OFFSET_DOMAIN_FACTOR)
|
|
|
|
|
if max_v is not None:
|
|
|
|
|
out["max_mem_offset_mhz"] = int(max_v)
|
|
|
|
|
out["max_mem_offset_mhz"] = int(max_v / _MEM_OFFSET_DOMAIN_FACTOR)
|
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
fn = _try_nvml_fn("nvmlDeviceGetMemClkMinMaxVfOffset")
|
|
|
|
@@ -395,8 +458,12 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
|
|
|
|
|
max_v = ctypes.c_int(0)
|
|
|
|
|
rc = fn(handle, ctypes.byref(min_v), ctypes.byref(max_v))
|
|
|
|
|
if rc == 0:
|
|
|
|
|
out["min_mem_offset_mhz"] = int(min_v.value)
|
|
|
|
|
out["max_mem_offset_mhz"] = int(max_v.value)
|
|
|
|
|
out["min_mem_offset_mhz"] = int(
|
|
|
|
|
min_v.value / _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
)
|
|
|
|
|
out["max_mem_offset_mhz"] = int(
|
|
|
|
|
max_v.value / _MEM_OFFSET_DOMAIN_FACTOR
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
log.debug("get_mem_offset_range: %s", exc)
|
|
|
|
|