+
Memory Clock Offset
diff --git a/nvcurve/hal/limits.py b/nvcurve/hal/limits.py
index 24c90f9..ad66ea3 100644
--- a/nvcurve/hal/limits.py
+++ b/nvcurve/hal/limits.py
@@ -8,6 +8,11 @@ Clock offsets use nvmlDeviceSetClockOffsets / nvmlDeviceGetClockOffsets
as a fallback when the new API is unavailable or returns an error.
set_clock_offsets accepts Optional values and only touches the domains
that are explicitly specified, leaving others unchanged on hardware.
+
+All public functions in this module use the *reported* clock domain for the
+memory offset — the domain of NVML_CLOCK_MEM, i.e. what nvidia-smi shows.
+The conversion to the driver's internal domain happens here (see
+_MEM_OFFSET_DOMAIN_FACTOR).
"""
import ctypes
@@ -179,6 +184,23 @@ _CLOCK_OFFSET_VER = (1 << 24) | ctypes.sizeof(_ClockOffset) # = 0x01000010 (16
_NVML_CLOCK_GRAPHICS = 0
_NVML_CLOCK_MEM = 2
+# Domain factor for the memory-V/F offset APIs (nvmlDeviceSetMemClkVfOffset /
+# nvmlDeviceSetClockOffsets(MEM) and their readback + range queries).
+#
+# Measured on RTX 5090, driver 615.78.08 (GDDR7/Blackwell): the driver takes
+# and reports the offset in a domain that is 2x the memory clock NVML
+# actually reports (NVML_CLOCK_MEM, what nvidia-smi shows). Applying +3000
+# moves the reported clock by only +1500 MHz; +6000 is the "real" +3000 in
+# nvidia-smi terms. (Cross-check: NvAPI GetClockBoostRanges says -1000/+3000
+# while nvmlDeviceGetMemClkMinMaxVfOffset says -2000/+6000 — the same range
+# in the two domains.)
+#
+# This module therefore converts at the NVML boundary:
+# set: nvml_value = reported_offset * _MEM_OFFSET_DOMAIN_FACTOR
+# get: reported_offset = nvml_value / _MEM_OFFSET_DOMAIN_FACTOR
+# range: reported = nvml_range / _MEM_OFFSET_DOMAIN_FACTOR
+_MEM_OFFSET_DOMAIN_FACTOR = 2
+
def _make_clock_offset(
clock_type: int, pstate: int = 0, offset_mhz: int = 0
@@ -208,10 +230,31 @@ def _try_nvml_fn(name: str):
return None
+def _nvml_raw_call(name: str, handle, info) -> int | None:
+ """Call an NVML function via its raw function pointer; return the real driver rc.
+
+ nvidia-ml-py's nvmlDeviceGet/SetClockOffsets wrappers discard the driver's
+ return code and always report success (verified in 13.615.71): on a driver
+ where the API errors, the zero-filled struct would be treated as valid and
+ the deprecated-API fallbacks would never run. Returns None when the raw
+ pointer is unavailable or cannot be called.
+ """
+ try:
+ get_ptr = getattr(pynvml, "_nvmlGetFunctionPointer", None)
+ if get_ptr is None:
+ return None
+ return int(get_ptr(name)(handle, ctypes.byref(info)))
+ except Exception as exc:
+ log.debug("_nvml_raw_call(%s): %s", name, exc)
+ return None
+
+
def get_clock_offsets(gpu_index: int = 0) -> dict:
"""Return GPC and memory clock offsets (MHz).
Keys: gpc_offset_mhz, mem_offset_mhz (both int or None on failure).
+ mem_offset_mhz is in the reported-clock domain (what nvidia-smi shows);
+ the driver's 2x-internal value is converted back here.
Calls nvmlDeviceGetClockOffsets once per clock domain (GRAPHICS, MEM).
"""
out: dict[str, int | None] = {"gpc_offset_mhz": None, "mem_offset_mhz": None}
@@ -220,12 +263,14 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
try:
handle = _get_handle(gpu_index)
- # Try pynvml wrapper first (nvidia-ml-py ≥ 12 exposes it correctly).
- # Fall back to ctypes-direct if pynvml doesn't have it.
+ # Prefer the raw function pointer: nvidia-ml-py's
+ # nvmlDeviceGetClockOffsets wrapper discards the driver's return code
+ # and always reports success (verified in 13.615.71), so an erroring
+ # driver would yield a zeroed struct we'd treat as valid and the
+ # deprecated fallback below would never run. The pynvml wrapper and
+ # the CDLL lookup remain as last resorts.
_pynvml_get = getattr(pynvml, "nvmlDeviceGetClockOffsets", None)
- fn_get = (
- _try_nvml_fn("nvmlDeviceGetClockOffsets") if _pynvml_get is None else None
- )
+ fn_get = _try_nvml_fn("nvmlDeviceGetClockOffsets")
used_new_api = False
for clock_type, key in (
@@ -234,14 +279,19 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
):
info = _make_clock_offset(clock_type, pstate=0)
try:
- if _pynvml_get is not None:
- rc = _pynvml_get(handle, ctypes.byref(info))
- elif fn_get is not None:
- rc = fn_get(handle, ctypes.byref(info))
- else:
- break
+ rc = _nvml_raw_call("nvmlDeviceGetClockOffsets", handle, info)
+ if rc is None:
+ if _pynvml_get is not None:
+ rc = _pynvml_get(handle, ctypes.byref(info))
+ elif fn_get is not None:
+ rc = fn_get(handle, ctypes.byref(info))
+ else:
+ break
if rc == 0:
- out[key] = int(info.clockOffsetMHz)
+ val = int(info.clockOffsetMHz)
+ if key == "mem_offset_mhz":
+ val = int(val / _MEM_OFFSET_DOMAIN_FACTOR)
+ out[key] = val
used_new_api = True
else:
log.debug(
@@ -262,9 +312,8 @@ def get_clock_offsets(gpu_index: int = 0) -> dict:
if hasattr(pynvml, "nvmlDeviceGetMemClkVfOffset"):
try:
res = pynvml.nvmlDeviceGetMemClkVfOffset(handle)
- out["mem_offset_mhz"] = int(
- res[0] if isinstance(res, (list, tuple)) else res
- )
+ raw = res[0] if isinstance(res, (list, tuple)) else res
+ out["mem_offset_mhz"] = int(raw / _MEM_OFFSET_DOMAIN_FACTOR)
except Exception as exc:
log.debug("nvmlDeviceGetMemClkVfOffset: %s", exc)
@@ -281,58 +330,69 @@ def set_clock_offsets(
"""Set clock offsets (MHz) for the specified domains only.
Pass None for a domain to leave it untouched on hardware.
+ mem_offset_mhz is in the reported-clock domain (what nvidia-smi shows);
+ it is multiplied by _MEM_OFFSET_DOMAIN_FACTOR before reaching the driver.
Calls nvmlDeviceSetClockOffsets once per requested domain (GRAPHICS, MEM).
Falls back to deprecated per-domain functions when the new API returns an
- error (e.g. NVML_ERROR_DEPRECATED=25 on Blackwell with driver 590.x).
+ error (e.g. NVML_ERROR_DEPRECATED=26 on Blackwell with driver 590.x).
"""
if gpc_offset_mhz is None and mem_offset_mhz is None:
return True, "OK"
if not _NVML_AVAILABLE:
return False, "NVML not available (install nvidia-ml-py)"
+ # Convert to the driver's internal domain (2x the reported clock on
+ # GDDR7/Blackwell; see _MEM_OFFSET_DOMAIN_FACTOR).
+ mem_api = (
+ mem_offset_mhz * _MEM_OFFSET_DOMAIN_FACTOR
+ if mem_offset_mhz is not None
+ else None
+ )
try:
handle = _get_handle(gpu_index)
domains = []
if gpc_offset_mhz is not None:
domains.append((_NVML_CLOCK_GRAPHICS, gpc_offset_mhz))
- if mem_offset_mhz is not None:
- domains.append((_NVML_CLOCK_MEM, mem_offset_mhz))
+ if mem_api is not None:
+ domains.append((_NVML_CLOCK_MEM, mem_api))
_pynvml_set = getattr(pynvml, "nvmlDeviceSetClockOffsets", None)
- fn_set = (
- _try_nvml_fn("nvmlDeviceSetClockOffsets") if _pynvml_set is None else None
- )
+ fn_set = _try_nvml_fn("nvmlDeviceSetClockOffsets")
- if _pynvml_set is not None or fn_set is not None:
- all_ok = True
- for clock_type, offset in domains:
- info = _make_clock_offset(clock_type, pstate=0, offset_mhz=offset)
- try:
+ # Same rationale as get_clock_offsets: the pynvml wrapper swallows the
+ # driver rc (always returns success), so go raw first. If every binding
+ # is missing, all_ok flips to False and we fall through to the
+ # deprecated path below.
+ all_ok = True
+ for clock_type, offset in domains:
+ info = _make_clock_offset(clock_type, pstate=0, offset_mhz=offset)
+ try:
+ rc = _nvml_raw_call("nvmlDeviceSetClockOffsets", handle, info)
+ if rc is None:
if _pynvml_set is not None:
rc = _pynvml_set(handle, ctypes.byref(info))
elif fn_set is not None:
rc = fn_set(handle, ctypes.byref(info))
- else:
- break
- if rc != 0:
- log.debug(
- "nvmlDeviceSetClockOffsets(type=%d) returned %d — trying fallback",
- clock_type,
- rc,
- )
- all_ok = False
- break
- except Exception as exc:
+ if rc is None or rc != 0:
log.debug(
- "nvmlDeviceSetClockOffsets(type=%d): %s — trying fallback",
+ "nvmlDeviceSetClockOffsets(type=%d) unavailable or returned %r — trying fallback",
clock_type,
- exc,
+ rc,
)
all_ok = False
break
- if all_ok:
- return True, "OK"
- # Non-zero rc (e.g. 25=DEPRECATED on Blackwell) — fall through to deprecated path.
+ except Exception as exc:
+ log.debug(
+ "nvmlDeviceSetClockOffsets(type=%d): %s — trying fallback",
+ clock_type,
+ exc,
+ )
+ all_ok = False
+ break
+ if all_ok:
+ return True, "OK"
+ # Non-zero rc (e.g. 26=DEPRECATED on Blackwell), or the new API is
+ # unavailable — fall through to the deprecated path.
# Deprecated per-domain fallback (works on Blackwell/driver 590.x).
errs = []
@@ -343,11 +403,11 @@ def set_clock_offsets(
pynvml.nvmlDeviceSetGpcClkVfOffset(handle, gpc_offset_mhz)
except Exception as exc:
errs.append(f"GPC: {exc}")
- if mem_offset_mhz is not None and hasattr(
+ if mem_api is not None and hasattr(
pynvml, "nvmlDeviceSetMemClkVfOffset"
):
try:
- pynvml.nvmlDeviceSetMemClkVfOffset(handle, mem_offset_mhz)
+ pynvml.nvmlDeviceSetMemClkVfOffset(handle, mem_api)
except Exception as exc:
errs.append(f"MEM: {exc}")
if errs:
@@ -366,10 +426,14 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
"""Return the min/max allowed memory clock offset (MHz).
Keys: min_mem_offset_mhz, max_mem_offset_mhz.
+ Values are in the reported-clock domain (what nvidia-smi shows); the
+ driver's 2x-internal range is converted down here.
Uses nvmlDeviceGetMemClkMinMaxVfOffset; falls back to observed RTX values.
"""
- # Observed RTX 5090 defaults (NvAPI GetClockBoostRanges says -1000/+3000).
- out: dict[str, int] = {"min_mem_offset_mhz": -2000, "max_mem_offset_mhz": 3000}
+ # Observed RTX 5090 range in the reported domain
+ # (NvAPI GetClockBoostRanges: -1000/+3000; the NVML range call returns the
+ # same as -2000/+6000 in its internal domain).
+ out: dict[str, int] = {"min_mem_offset_mhz": -1000, "max_mem_offset_mhz": 3000}
if not _NVML_AVAILABLE:
return out
try:
@@ -378,15 +442,14 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
if hasattr(pynvml, "nvmlDeviceGetMemClkMinMaxVfOffset"):
result = pynvml.nvmlDeviceGetMemClkMinMaxVfOffset(handle)
if isinstance(result, (list, tuple)) and len(result) >= 2:
- out["min_mem_offset_mhz"] = int(result[0])
- out["max_mem_offset_mhz"] = int(result[1])
+ min_v, max_v = int(result[0]), int(result[1])
else:
min_v = getattr(result, "minOffset", None)
max_v = getattr(result, "maxOffset", None)
- if min_v is not None:
- out["min_mem_offset_mhz"] = int(min_v)
- if max_v is not None:
- out["max_mem_offset_mhz"] = int(max_v)
+ if min_v is not None:
+ out["min_mem_offset_mhz"] = int(min_v / _MEM_OFFSET_DOMAIN_FACTOR)
+ if max_v is not None:
+ out["max_mem_offset_mhz"] = int(max_v / _MEM_OFFSET_DOMAIN_FACTOR)
return out
fn = _try_nvml_fn("nvmlDeviceGetMemClkMinMaxVfOffset")
@@ -395,8 +458,12 @@ def get_mem_offset_range(gpu_index: int = 0) -> dict:
max_v = ctypes.c_int(0)
rc = fn(handle, ctypes.byref(min_v), ctypes.byref(max_v))
if rc == 0:
- out["min_mem_offset_mhz"] = int(min_v.value)
- out["max_mem_offset_mhz"] = int(max_v.value)
+ out["min_mem_offset_mhz"] = int(
+ min_v.value / _MEM_OFFSET_DOMAIN_FACTOR
+ )
+ out["max_mem_offset_mhz"] = int(
+ max_v.value / _MEM_OFFSET_DOMAIN_FACTOR
+ )
except Exception as exc:
log.debug("get_mem_offset_range: %s", exc)