From dc54d3f5fca98e25d38559f97cdb7e8800ee0b36 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 18 Oct 2026 23:25:22 +0000 Subject: [PATCH 1/4] test(bindings): work around Orin NVML field-values bug Seed the field status with an invalid sentinel so a successful native no-op cannot accidentally pass on recycled memory. Xfail when Orin leaves the sentinel unchanged, preserving normal checks on fixed drivers and other devices. Label the workaround explicitly in the source comment and xfail reason. --- cuda_bindings/tests/nvml/test_nvlink.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cuda_bindings/tests/nvml/test_nvlink.py b/cuda_bindings/tests/nvml/test_nvlink.py index 1ea9e25dfa7..08adb1290d5 100644 --- a/cuda_bindings/tests/nvml/test_nvlink.py +++ b/cuda_bindings/tests/nvml/test_nvlink.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pytest from cuda.bindings import nvml @@ -17,7 +18,12 @@ def test_nvlink_get_link_count(all_devices): fields = nvml.FieldValue(1) fields[0].field_id = nvml.FieldId.DEV_NVLINK_LINK_COUNT + fields[0].scope_id = 0 + # WORKAROUND: Orin NVML can return success without populating field values. + fields[0].nvml_return = -1 value = nvml.device_get_field_values(device, fields)[0] + if value.nvml_return == -1 and "Orin" in nvml.device_get_name(device): + pytest.xfail("Orin NVML bug: nvmlDeviceGetFieldValues returns success without populating the result") assert value.nvml_return == nvml.Return.SUCCESS or value.nvml_return == nvml.Return.ERROR_NOT_SUPPORTED, ( f"Unexpected return {value.nvml_return} for link count field query" ) From 412659b856b8c1d7086bdef9351f66e3b4cecabd Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 18 Oct 2026 20:43:36 -0700 Subject: [PATCH 2/4] Add nvbug number to xfail for easy reference --- cuda_bindings/tests/nvml/test_nvlink.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cuda_bindings/tests/nvml/test_nvlink.py b/cuda_bindings/tests/nvml/test_nvlink.py index 08adb1290d5..658dd4434e5 100644 --- a/cuda_bindings/tests/nvml/test_nvlink.py +++ b/cuda_bindings/tests/nvml/test_nvlink.py @@ -23,7 +23,9 @@ def test_nvlink_get_link_count(all_devices): fields[0].nvml_return = -1 value = nvml.device_get_field_values(device, fields)[0] if value.nvml_return == -1 and "Orin" in nvml.device_get_name(device): - pytest.xfail("Orin NVML bug: nvmlDeviceGetFieldValues returns success without populating the result") + pytest.xfail( + "Orin NVML nvbug 6952048: nvmlDeviceGetFieldValues returns success without populating the result" + ) assert value.nvml_return == nvml.Return.SUCCESS or value.nvml_return == nvml.Return.ERROR_NOT_SUPPORTED, ( f"Unexpected return {value.nvml_return} for link count field query" ) From 0bd66377c9a8a05abe8b6ab79eb3dfaa36589116 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 19 Oct 2026 17:03:52 -0700 Subject: [PATCH 3/4] Regenerate NVML bindings with per-query error initialization --- cuda_bindings/cuda/bindings/nvml.pyx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cuda_bindings/cuda/bindings/nvml.pyx b/cuda_bindings/cuda/bindings/nvml.pyx index 4a3c2ca260b..6bcd74b1ee2 100644 --- a/cuda_bindings/cuda/bindings/nvml.pyx +++ b/cuda_bindings/cuda/bindings/nvml.pyx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated across versions from 12.9.1 to 13.4.1. Do not modify it directly. -# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=341606be9fe4e6d759ff42b1ac1e10ac2e68e9412f550de5781e57183e218297 +# CYTHON-BINDINGS-GENERATED-DO-NOT-MODIFY-THIS-FILE: format=1; content-sha256=0e14261df396c087ab467037853a05290c6a90be1f3a69d6f8f2f9aea22e8e64 # <<<< PREAMBLE CONTENT >>>> @@ -36679,6 +36679,7 @@ cdef FieldValue _cast_field_values(values): values_[i].scope_id = v[1] elif isinstance(v, int): values_[i].field_id = v + values_[i].scope_id = 0 else: raise ValueError("Each entry must be an integer field ID, or a tuple of (field ID, scope ID)") return values_ @@ -36691,16 +36692,24 @@ cpdef object device_get_field_values(intptr_t device, values): device (intptr_t): The device handle of the GPU to request field values for. values (FieldValue): Array of FieldValue specifying what to retrieve. + Entries left unpopulated by NVML report ``ERROR_UNKNOWN`` in ``nvml_return``. + .. seealso:: `nvmlDeviceGetFieldValues` """ cdef FieldValue values_ = _cast_field_values(values) cdef nvmlFieldValue_t *ptr = values_._get_ptr() cdef unsigned int valuesCount = len(values) + cdef unsigned int i # Passing a valuesCount of 0 to nvmlDeviceGetFieldValues returns NVML_INVALID_ARGUMENT if valuesCount == 0: return values_ + # nvbug 6952048: native NVML can return success without populating an entry. + # Reset every query so untouched entries cannot retain stale success statuses. + for i in range(valuesCount): + ptr[i].nvmlReturn = NVML_ERROR_UNKNOWN + with nogil: __status__ = nvmlDeviceGetFieldValues(device, valuesCount, ptr) check_status(__status__) From 2a11713dfd53da6a0789f9bb0c4b83539aa7e3d7 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 19 Oct 2026 17:06:16 -0700 Subject: [PATCH 4/4] Test NVML field queries with untouched and partial native results --- cuda_bindings/tests/nvml/test_field_values.py | 207 ++++++++++++++++++ cuda_bindings/tests/nvml/test_nvlink.py | 28 +-- 2 files changed, 219 insertions(+), 16 deletions(-) create mode 100644 cuda_bindings/tests/nvml/test_field_values.py diff --git a/cuda_bindings/tests/nvml/test_field_values.py b/cuda_bindings/tests/nvml/test_field_values.py new file mode 100644 index 00000000000..17bec49abd8 --- /dev/null +++ b/cuda_bindings/tests/nvml/test_field_values.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import sys +import textwrap +from pathlib import Path + +import pytest +from cuda_python_test_helpers.subprocess_runner import run_python_snippet +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler + +from cuda.pathfinder import find_nvidia_header_directory + +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Native NVML shim uses the Linux loader") + + +_SHIM_SOURCE = r""" +#include +#include + +static unsigned int query_count; + +unsigned int test_get_query_count(void) { + return query_count; +} + +const char *nvmlErrorString(nvmlReturn_t status) { + (void)status; + return "native NVML test error"; +} + +nvmlReturn_t nvmlDeviceGetFieldValues(nvmlDevice_t device, int count, + nvmlFieldValue_t *values) { + ++query_count; + if ((uintptr_t)device == 2) { + return NVML_ERROR_NO_PERMISSION; + } + for (int i = 0; i < count; ++i) { + switch (values[i].fieldId) { + case 1: /* Successful native no-op. */ + break; + case 2: + values[i].timestamp = 12345; + values[i].latencyUsec = 7; + values[i].valueType = NVML_VALUE_TYPE_UNSIGNED_INT; + values[i].value.uiVal = 17; + values[i].nvmlReturn = NVML_SUCCESS; + break; + case 3: + values[i].nvmlReturn = NVML_ERROR_NOT_SUPPORTED; + break; + case 4: + values[i].nvmlReturn = NVML_ERROR_UNKNOWN; + break; + case 5: + values[i].valueType = NVML_VALUE_TYPE_UNSIGNED_INT; + values[i].value.uiVal = values[i].scopeId; + values[i].nvmlReturn = NVML_SUCCESS; + break; + } + } + return NVML_SUCCESS; +} +""" + +_SHIM_SETUP = """ +import ctypes +import os + +shim = ctypes.CDLL(os.environ["CUDA_PYTHON_TEST_NVML_SHIM"], mode=ctypes.RTLD_GLOBAL) +from cuda.bindings import nvml +""" + + +@pytest.fixture(scope="module") +def ctx(): + """Override the parent CUDA-context fixture for these native-shim tests.""" + + +@pytest.fixture(scope="module") +def field_values_shim(tmp_path_factory): + header_directory = find_nvidia_header_directory("cudart") + if header_directory is None or not (Path(header_directory) / "nvml.h").is_file(): + pytest.skip("NVML header is required to compile the native test shim") + + compiler = new_compiler() + customize_compiler(compiler) + if shutil.which(compiler.compiler[0]) is None: + pytest.skip("C compiler is required to compile the native test shim") + directory = tmp_path_factory.mktemp("nvml-field-values") + source = directory / "field_values.c" + source.write_text(_SHIM_SOURCE, encoding="utf-8") + objects = compiler.compile([str(source)], output_dir=str(directory), include_dirs=[header_directory]) + library = directory / "libnvidia-ml.so.1" + compiler.link_shared_object(objects, str(library)) + return library + + +def _run_with_shim(library, code): + # A fresh process keeps the native symbol cache and shim out of other tests. + run_python_snippet( + _SHIM_SETUP + textwrap.dedent(code), + timeout=30, + extra_env={ + "CUDA_PYTHON_TEST_NVML_SHIM": str(library), + "LD_LIBRARY_PATH": os.pathsep.join(filter(None, (str(library.parent), os.environ.get("LD_LIBRARY_PATH")))), + }, + ) + + +@pytest.mark.parametrize("initial_status", [0, 3, 5, 12345]) +@pytest.mark.agent_authored(model="gpt-6") +def test_unpopulated_field_values_report_unknown(field_values_shim, initial_status): + _run_with_shim( + field_values_shim, + f""" + fields = nvml.FieldValue(2) + fields.field_id = [1, 1] + fields.scope_id = [0, 11] + fields.nvml_return = {initial_status} + for _ in range(2): + result = nvml.device_get_field_values(1, fields) + assert list(result.nvml_return) == [nvml.Return.ERROR_UNKNOWN] * 2 + assert list(result.field_id) == [1, 1] + assert list(result.scope_id) == [0, 11] + fields.nvml_return = nvml.Return.SUCCESS + """, + ) + + +@pytest.mark.agent_authored(model="gpt-6") +def test_field_values_preserve_native_results_and_scopes(field_values_shim): + _run_with_shim( + field_values_shim, + """ + import numpy as np + + fields = nvml.FieldValue(4) + fields.field_id = [2, 3, 1, 4] + fields.scope_id = [7, 8, 9, 10] + fields.nvml_return = nvml.Return.SUCCESS + result = nvml.device_get_field_values(1, fields) + assert list(result.nvml_return) == [ + nvml.Return.SUCCESS, nvml.Return.ERROR_NOT_SUPPORTED, + nvml.Return.ERROR_UNKNOWN, nvml.Return.ERROR_UNKNOWN, + ] + assert result[0].timestamp == 12345 + assert result[0].latency_usec == 7 + assert result[0].value_type == nvml.ValueType.UNSIGNED_INT + assert result[0].value.ui_val[0] == 17 + assert list(result.field_id) == [2, 3, 1, 4] + assert list(result.scope_id) == [7, 8, 9, 10] + + original_empty = np.empty + def poisoned_empty(*args, **kwargs): + array = original_empty(*args, **kwargs) + array.view(np.uint8).fill(0xA5) + return array + + np.empty = poisoned_empty + try: + result = nvml.device_get_field_values(1, [5, 5]) + finally: + np.empty = original_empty + assert list(result.scope_id) == [0, 0] + assert list(result.nvml_return) == [nvml.Return.SUCCESS] * 2 + assert list(result.value.ui_val) == [0, 0] + + result = nvml.device_get_field_values(1, [(5, 19), (5, 43)]) + assert list(result.scope_id) == [19, 43] + assert list(result.nvml_return) == [nvml.Return.SUCCESS] * 2 + assert list(result.value.ui_val) == [19, 43] + + for values in ([1], [(1, 11)]): + result = nvml.device_get_field_values(1, values) + assert result[0].nvml_return == nvml.Return.ERROR_UNKNOWN + """, + ) + + +@pytest.mark.agent_authored(model="gpt-6") +def test_field_values_preserve_aggregate_error(field_values_shim): + _run_with_shim( + field_values_shim, + """ + import pytest + + with pytest.raises(nvml.NoPermissionError) as caught: + nvml.device_get_field_values(2, [1]) + assert caught.value.status == nvml.Return.ERROR_NO_PERMISSION + """, + ) + + +@pytest.mark.agent_authored(model="gpt-6") +def test_empty_field_values_do_not_call_native_query(field_values_shim): + _run_with_shim( + field_values_shim, + """ + assert len(nvml.device_get_field_values(2, [])) == 0 + assert len(nvml.device_get_field_values(2, nvml.FieldValue(0))) == 0 + assert shim.test_get_query_count() == 0 + """, + ) diff --git a/cuda_bindings/tests/nvml/test_nvlink.py b/cuda_bindings/tests/nvml/test_nvlink.py index 658dd4434e5..254d7c7a48c 100644 --- a/cuda_bindings/tests/nvml/test_nvlink.py +++ b/cuda_bindings/tests/nvml/test_nvlink.py @@ -6,7 +6,8 @@ from cuda.bindings import nvml -def test_nvlink_get_link_count(all_devices): +@pytest.mark.agent_authored(model="gpt-6") +def test_nvlink_get_link_count(all_devices, subtests): """ Checks that the link count of the device is same. """ @@ -19,19 +20,14 @@ def test_nvlink_get_link_count(all_devices): fields = nvml.FieldValue(1) fields[0].field_id = nvml.FieldId.DEV_NVLINK_LINK_COUNT fields[0].scope_id = 0 - # WORKAROUND: Orin NVML can return success without populating field values. - fields[0].nvml_return = -1 - value = nvml.device_get_field_values(device, fields)[0] - if value.nvml_return == -1 and "Orin" in nvml.device_get_name(device): - pytest.xfail( - "Orin NVML nvbug 6952048: nvmlDeviceGetFieldValues returns success without populating the result" - ) - assert value.nvml_return == nvml.Return.SUCCESS or value.nvml_return == nvml.Return.ERROR_NOT_SUPPORTED, ( - f"Unexpected return {value.nvml_return} for link count field query" - ) + expected_statuses = {nvml.Return.SUCCESS, nvml.Return.ERROR_NOT_SUPPORTED} + if nvml.device_get_name(device) == "Orin (nvgpu)": + # nvbug 6952048: the binding reports UNKNOWN for untouched native output. + expected_statuses.add(nvml.Return.ERROR_UNKNOWN) - # Use the alternative argument to device_get_field_values - value = nvml.device_get_field_values(device, [nvml.FieldId.DEV_NVLINK_LINK_COUNT])[0] - assert value.nvml_return == nvml.Return.SUCCESS or value.nvml_return == nvml.Return.ERROR_NOT_SUPPORTED, ( - f"Unexpected return {value.nvml_return} for link count field query" - ) + for input_kind, values in (("buffer", fields), ("field_ids", [nvml.FieldId.DEV_NVLINK_LINK_COUNT])): + with subtests.test(device=device, input_kind=input_kind): + value = nvml.device_get_field_values(device, values)[0] + assert value.nvml_return in expected_statuses, ( + f"Unexpected return {value.nvml_return} for link count field query" + )