From 09eb52fe0ad713af1ed28269f5f29071b6cb1a5e Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:50:02 +0800 Subject: [PATCH 01/12] refactor(detector): split the query surface and retire the physical index switch - add `Detector.detect_info` / `detect_usage` / `detect(usage=True)`, both halves abstract, so a vendor cannot half implement the split and have `detect` look like it measured something - add `merge_devices_usage`, joining usage onto information by UUID, MIG entries in `appendix["mig_devices"]` included, mirroring the operator's `MonitorAccelerator`, which returns a UUID-keyed metrics list consumers join by identity, never by index - drop an ambiguous UUID from that join instead of keying a dict by it, which collapsed last-wins and wrote one card's utilization, memory, temperature and power onto every card answering the same id - keep the information query's `memory_status` when the usage entry left it UNKNOWN: no vendor's health helper returns UNKNOWN, so such an entry never read health, and writing it erased a verdict the CLI renders as ERR - thread `usage` through `detect_devices`, detecting topologies without it, and keep the inventory when only the usage query fails -- a raising detector is logged and skipped, so one failed metric call reported a host of healthy cards as none - drop `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY`, recording the driver's physical number in the appendix instead, as the operator does, and keep `/dev/davinci{N}` and `/dev/iluvatar{N}` on that number so no device node path changes meaning - add `get_pci_device_name`, mirroring the operator's pci.ids lookup, and delete `get_physical_function_by_bdf`, whose last caller goes with the vGPU classification - add `detect --no-usage`, rendering the unmeasured table columns as N/A and omitting the five usage-only JSON keys, MIG instances included, rather than emitting a zero that reads like a measurement Signed-off-by: thxCode --- gpustack_runtime/cmds/detector.py | 59 +++++- gpustack_runtime/detector/__init__.py | 11 +- gpustack_runtime/detector/__types__.py | 180 ++++++++++++++++-- gpustack_runtime/detector/__utils__.py | 153 +++++++++++++-- gpustack_runtime/envs.py | 10 - .../detector/test_mig_devices.py | 10 +- 6 files changed, 370 insertions(+), 53 deletions(-) diff --git a/gpustack_runtime/cmds/detector.py b/gpustack_runtime/cmds/detector.py index 8b109df..2a0349d 100644 --- a/gpustack_runtime/cmds/detector.py +++ b/gpustack_runtime/cmds/detector.py @@ -26,6 +26,7 @@ class DetectDevicesSubCommand(SubCommand): format: str = "table" watch: int = 0 + no_usage: bool = False @staticmethod def register(parser: _SubParsersAction): @@ -49,19 +50,26 @@ def register(parser: _SubParsersAction): help="Continuously watch for GPU in intervals of N seconds", ) + detect_parser.add_argument( + "--no-usage", + action="store_true", + help="Report inventory only, without querying utilization, temperature or power", + ) + detect_parser.set_defaults(func=DetectDevicesSubCommand) def __init__(self, args: Namespace): self.format = args.format self.watch = args.watch + self.no_usage = args.no_usage def run(self): while True: - devs: Devices = detect_devices(fast=False) + devs: Devices = detect_devices(fast=False, usage=not self.no_usage) print("\033[2J\033[H", end="") match self.format.lower(): case "json": - print(format_devices_json(devs)) + print(format_devices_json(devs, usage=not self.no_usage)) case _: # Group devices by manufacturer. group_devs = group_devices_by_manufacturer(devs) @@ -70,7 +78,9 @@ def run(self): else: # Print each group separately. for devs in group_devs.values(): - print(format_devices_table(devs)) + print( + format_devices_table(devs, usage=not self.no_usage), + ) if not self.watch: break time.sleep(self.watch) @@ -134,11 +144,37 @@ def run(self): print(os.linesep.join(legend_lines)) -def format_devices_json(devs: Devices) -> str: - return json.dumps([dev.to_dict() for dev in devs], indent=2) +_USAGE_ONLY_KEYS = ( + "cores_utilization", + "memory_used", + "memory_utilization", + "temperature", + "power_used", +) +""" +The keys only the usage query fills. `memory_status` is deliberately absent: +the information query reports it too, which is why the table keeps Status. +""" + + +def format_devices_json(devs: Devices, usage: bool = True) -> str: + devs_dict = [dev.to_dict() for dev in devs] + + if not usage: + # Without the usage query these fields hold their defaults, and a + # serialized 0 reads as a real idle measurement. The table drops them + # for that reason, and a machine-readable consumer is the more likely of + # the two to act on it -- so absent, meaning unmeasured. + for dev_dict in devs_dict: + mig_devs_dict = (dev_dict.get("appendix") or {}).get("mig_devices") or [] + for d in [dev_dict, *mig_devs_dict]: + for key in _USAGE_ONLY_KEYS: + d.pop(key, None) + + return json.dumps(devs_dict, indent=2) -def format_devices_table(devs: Devices) -> str: +def format_devices_table(devs: Devices, usage: bool = True) -> str: if not devs: return "No GPUs detected." @@ -150,9 +186,14 @@ def format_devices_table(devs: Devices) -> str: row = [ str(dev.index), dev.name if dev.name else "N/A", - f"{dev.memory_used}MiB / {dev.memory}MiB", - f"{dev.cores_utilization}%", - f"{dev.temperature}C" if dev.temperature is not None else "N/A", + # Without the usage query these fields hold their defaults, and a + # rendered 0 would read as a real idle measurement. Total memory + # and the status stay, being reported by the information query. + f"{dev.memory_used}MiB / {dev.memory}MiB" + if usage + else f"N/A / {dev.memory}MiB", + f"{dev.cores_utilization}%" if usage else "N/A", + f"{dev.temperature}C" if usage and dev.temperature is not None else "N/A", dev.compute_capability if dev.compute_capability else "N/A", "OK" if dev.memory_status == DeviceMemoryStatusEnum.HEALTHY else "ERR", ] diff --git a/gpustack_runtime/detector/__init__.py b/gpustack_runtime/detector/__init__.py index 7a7d6db..b21ae64 100644 --- a/gpustack_runtime/detector/__init__.py +++ b/gpustack_runtime/detector/__init__.py @@ -141,6 +141,7 @@ def detect_backend( def detect_devices( fast: bool = True, manufacturer: ManufacturerEnum = None, + usage: bool = True, ) -> Devices: """ Detect all available devices. @@ -152,6 +153,8 @@ def detect_devices( manufacturer: Manufacturer to filter the detection, implies `fast=True`. If None, detect all available manufacturers. + usage: + If True, fetch the devices' usage as well. Returns: A list of detected devices. @@ -165,7 +168,7 @@ def detect_devices( det = _DETECTORS_MAP.get(manufacturer) if det and det.is_supported(): try: - return det.detect() + return det.detect(usage=usage) except Exception: detect_target = envs.GPUSTACK_RUNTIME_DETECT.lower() if detect_target == det.name: @@ -180,7 +183,7 @@ def detect_devices( continue try: - if devs := det.detect(): + if devs := det.detect(usage=usage): devices.extend(devs) if fast and devices: return devices @@ -219,7 +222,9 @@ def get_devices_topologies( """ group = False if not devices: - devices = detect_devices(fast=fast, manufacturer=manufacturer) + # Topology keys off identity and the appendix only, so the usage query + # is not worth its cost here. + devices = detect_devices(fast=fast, manufacturer=manufacturer, usage=False) if not devices: return [] group = not fast diff --git a/gpustack_runtime/detector/__types__.py b/gpustack_runtime/detector/__types__.py index f28e0fc..32b854f 100644 --- a/gpustack_runtime/detector/__types__.py +++ b/gpustack_runtime/detector/__types__.py @@ -1,5 +1,6 @@ from __future__ import annotations as __future_annotations__ +import logging from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum @@ -8,6 +9,10 @@ from dataclasses_json import dataclass_json +from ..logging import debug_log_exception, debug_log_warning + +logger = logging.getLogger(__name__) + class ManufacturerEnum(str, Enum): """ @@ -157,11 +162,9 @@ class Device: """ index: int = 0 """ - Index of the device. - If GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY is set to 1, - this will be the physical index of the device. - Otherwise, it will be the logical index of the device. - Physical index is adapted to non-virtualized devices. + Index of the device, as the detector enumerates it. + Driver-physical numbering, which a device node path or a vendor tool needs, + lives in `appendix` instead, e.g. `minor_number` or `card_id`/`physical_id`. """ name: str = "" """ @@ -471,11 +474,10 @@ def index_mig_devices( A MIG device carries no driver-side inventory index, so its index is synthetic: every card owns a block of `slots` indexes, and the blocks - start above the largest index the physical cards report. The reported - index may be the card's minor number - (`GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY`), which is not bound by - the card count, hence the offset is measured from the reported indexes - instead of the count. Sizing a block by the slots a card can host, rather + start above the largest index the physical cards report. A detector's + reported index is not necessarily zero-based and contiguous — Ascend + reports the DCMI logic id — hence the offset is measured from the reported + indexes instead of the card count. Sizing a block by the slots a card can host, rather than by the MIG devices it currently has, keeps a card's numbering independent of its neighbours: partitioning one card never renumbers another's MIG devices. @@ -497,6 +499,113 @@ def index_mig_devices( mig["index"] += base + dev_idx * slots +_DEVICE_USAGE_FIELDS = ( + "cores_utilization", + "memory_used", + "memory_utilization", + "memory_status", + "temperature", + "power_used", +) +""" +The fields the usage query owns, i.e. everything a detector must re-read to +refresh a device it already knows. `memory_status` belongs to both queries, +mirroring the operator, which reports health from `DetectAccelerator` and +`MonitorAccelerator` alike. +""" + + +def merge_devices_usage( + devices: Devices | None, + usages: Devices | None, +) -> Devices | None: + """ + Merge the given usage into the given devices in place, matched by UUID. + + The usage query returns devices of its own, keyed by UUID, which this + joins into the devices to refresh: the operator does the same, its + `MonitorAccelerator` returning a separate metrics list that every consumer + joins by device identity and never by index, as an index is not stable + across a re-detection. + + MIG devices are refreshed as well: they live in the card's + `appendix["mig_devices"]` and carry their own UUID, so a usage entry + matching one is merged into that entry. + + Args: + devices: + The devices to refresh. + usages: + The devices carrying the usage to merge. + + Returns: + The given devices, refreshed. + + """ + if not devices or not usages: + return devices + + # The join is only as good as the identities behind it, and a driver + # answering the same id for every card is how that breaks: taking either + # entry would write one card's utilization, memory and temperature onto + # another. An ambiguous id is therefore dropped rather than guessed at, + # leaving those cards with what the information query read. + usages_map: dict[str, Device] = {} + ambiguous_uuids: set[str] = set() + for usage in usages: + if not usage.uuid: + continue + if usage.uuid in usages_map: + ambiguous_uuids.add(usage.uuid) + continue + usages_map[usage.uuid] = usage + for uuid in ambiguous_uuids: + debug_log_warning( + logger, + "Skipping usage of uuid %s, reported by more than one device", + uuid, + ) + del usages_map[uuid] + + for dev in devices: + if usage := usages_map.get(dev.uuid): + for field in _usage_fields_to_merge(usage): + setattr(dev, field, getattr(usage, field)) + + # Keyed by the instance's own UUID, so what lands in a MIG entry is the + # usage query's reading for that instance -- the entry is written, never + # read from. + for mig_dev in (dev.appendix or {}).get("mig_devices") or []: + if mig_usage := usages_map.get(mig_dev.get("uuid")): + for field in _usage_fields_to_merge(mig_usage): + mig_dev[field] = getattr(mig_usage, field) + + return devices + + +def _usage_fields_to_merge(usage: Device) -> tuple[str, ...]: + """ + Select the fields of the given usage entry that are worth writing over a device. + + Args: + usage: + The device carrying the usage to merge. + + Returns: + The names of the fields to copy. + + """ + if usage.memory_status != DeviceMemoryStatusEnum.UNKNOWN: + return _DEVICE_USAGE_FIELDS + + # No vendor's health helper returns UNKNOWN: they answer HEALTHY or + # UNHEALTHY, and HEALTHY when the check is switched off. So an entry carrying + # UNKNOWN never read health, and writing it would erase the verdict the + # information query found -- which the CLI renders as ERR, exactly as it + # renders UNHEALTHY. + return tuple(field for field in _DEVICE_USAGE_FIELDS if field != "memory_status") + + class Detector(ABC): """ Base class for all detectors. @@ -537,15 +646,62 @@ def name(self) -> str: return str(self.manufacturer) @abstractmethod - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: + """ + Detect devices' inventory, without usage metrics. + + Returns: + A list of detected Device objects, or None if not supported. + + """ + raise NotImplementedError + + @abstractmethod + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of the given devices, merged into them in place. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + """ + raise NotImplementedError + + def detect(self, usage: bool = True) -> Devices | None: """ Detect devices and return a list of Device objects. + Args: + usage: + Whether to fetch the devices' usage as well. + Returns: A list of detected Device objects, or None if detection fails. """ - raise NotImplementedError + devices = self.detect_info() + if usage and devices: + try: + self.detect_usage(devices) + except Exception: + # The inventory is already in hand, and a card exists whether or + # not its metrics could be read. Letting this propagate costs + # every device of the vendor -- detect_devices logs and moves on + # -- so a host with eight healthy cards reports none because one + # metric query failed. The usage fields keep what the + # information query read. + debug_log_exception( + logger, + "Failed to fetch %s devices usage, reporting information only", + self.manufacturer, + ) + return devices def get_topology(self, devices: Devices | None = None) -> Topology | None: """ diff --git a/gpustack_runtime/detector/__utils__.py b/gpustack_runtime/detector/__utils__.py index e50b742..e3a2e6d 100644 --- a/gpustack_runtime/detector/__utils__.py +++ b/gpustack_runtime/detector/__utils__.py @@ -1013,24 +1013,149 @@ def bitmask_to_str(bitmask_list: list) -> str: return list_to_str_range(sorted(bits_lists)) -def get_physical_function_by_bdf(bdf: str) -> str: +_PCI_IDS_PATHS: tuple[str, ...] = ( + "/usr/share/hwdata/pci.ids", + "/usr/share/pci.ids", + "/usr/share/misc/pci.ids", +) +""" +Candidate locations of the PCI ID database, +in the order the operator's GetPCIDeviceNames probes them. +""" + + +def _normalize_pci_id(value: int | str | None) -> str: """ - Get the physical function BDF for a given PCI device BDF address. + Normalize a PCI ID to the lowercase hexadecimal form the PCI ID database uses. Args: - bdf: - The PCI device BDF address (e.g., "0000:00:1f.0"). + value: + The PCI ID, e.g. 0x1002, "0x1002" or "1002". + sysfs reports it prefixed, the vendor SMI libraries report an integer. Returns: - The physical function BDF if found, otherwise returns the original BDF. + The normalized PCI ID, or an empty string if there is none. """ - if bdf: - with contextlib.suppress(Exception): - dev_path = Path(f"/sys/bus/pci/devices/{bdf}") - if dev_path.exists(): - physfn_path = dev_path / "physfn" - if physfn_path.exists(): - physfn_realpath = physfn_path.resolve() - return physfn_realpath.name - return bdf + if value is None: + return "" + if isinstance(value, int): + return f"{value:04x}" + return value.strip().lower().removeprefix("0x") + + +@lru_cache +def _load_pci_device_names( + vendor: str, +) -> dict[str, tuple[str, dict[tuple[str, str], str]]]: + """ + Parse the PCI ID database for one vendor. + + Mirrors the parsing of the operator's GetPCIDeviceNames: a vendor line + starts at column 0, a device line behind one tab, a subsystem line behind + two. The database also holds a device class table ("C 03 Display + controller") and comments, which are read as vendor lines and never match + a real vendor, so they gate off the same way an unrequested vendor does. + + Args: + vendor: + The normalized PCI vendor ID to parse for. + + Returns: + A mapping of PCI device ID to the device's name and its subsystem + names, keyed by (subsystem vendor ID, subsystem device ID). + Empty if the database or the vendor is not found. + + """ + names: dict[str, tuple[str, dict[tuple[str, str], str]]] = {} + if not vendor: + return names + + path = next((p for p in map(Path, _PCI_IDS_PATHS) if p.exists()), None) + if not path: + return names + + in_vendor = False + device = "" + with ( + contextlib.suppress(OSError), + path.open("r", encoding="utf-8", errors="ignore") as f, + ): + for raw_line in f: + line = raw_line.rstrip("\n") + if not line: + continue + + fields = line.split() + + # A vendor line, which switches the section being parsed. A line + # holding a single field leaves the section as it is, as the + # operator does. + if not line.startswith("\t"): + if len(fields) > 1: + in_vendor = fields[0].lower() == vendor + device = "" + continue + + if not in_vendor: + continue + + # A device line. + if not line.startswith("\t\t"): + if len(fields) > 1: + device = fields[0].lower() + names[device] = (" ".join(fields[1:]), {}) + continue + + # A subsystem line, belonging to the device line above it. + if device and len(fields) > 2: + names[device][1][(fields[0].lower(), fields[1].lower())] = " ".join( + fields[2:], + ) + + return names + + +def get_pci_device_name( + vendor: int | str, + device: int | str, + subvendor: int | str = "", + subdevice: int | str = "", +) -> str: + """ + Get the name of a PCI device from the local PCI ID database. + + Mirrors the operator's GetPCIDeviceNames/GetName, which prefers the + subsystem name over the device name: a board vendor's name for the card is + more precise than the chip's. + + Args: + vendor: + The PCI vendor ID, e.g. 0x1002, "0x1002" or "1002". + device: + The PCI device ID. + subvendor: + The PCI subsystem vendor ID, if any. + subdevice: + The PCI subsystem device ID, if any. + + Returns: + The name of the device, + or an empty string if the database or the device is not found. + + """ + vendor = _normalize_pci_id(vendor) + device = _normalize_pci_id(device) + if not vendor or not device: + return "" + + entry = _load_pci_device_names(vendor).get(device) + if not entry: + return "" + + name, subnames = entry + subvendor = _normalize_pci_id(subvendor) + subdevice = _normalize_pci_id(subdevice) + if subvendor and subdevice: + return subnames.get((subvendor, subdevice), name) + return name diff --git a/gpustack_runtime/envs.py b/gpustack_runtime/envs.py index 17782bc..3cfa266 100644 --- a/gpustack_runtime/envs.py +++ b/gpustack_runtime/envs.py @@ -55,10 +55,6 @@ e.g `{"cuda": "nvidia.com/devices", "rocm": "amd.com/devices"}`. Used to map the gpustack-runner's backend name to the corresponding resource key. """ - GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: bool = True - """ - Use physical index priority at detecting devices. - """ ## Deployer GPUSTACK_RUNTIME_DEPLOY: str | None = None """ @@ -411,12 +407,6 @@ "hggc=alibabacloud.com/devices;", ), ), - "GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY": lambda: to_bool( - getenv( - "GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY", - "1", - ), - ), ## Deployer "GPUSTACK_RUNTIME_DEPLOY": lambda: getenv( "GPUSTACK_RUNTIME_DEPLOY", diff --git a/tests/gpustack_runtime/detector/test_mig_devices.py b/tests/gpustack_runtime/detector/test_mig_devices.py index aec2131..02c9fd1 100644 --- a/tests/gpustack_runtime/detector/test_mig_devices.py +++ b/tests/gpustack_runtime/detector/test_mig_devices.py @@ -29,7 +29,7 @@ def _mig(slot: int, uuid: str) -> dict: "name": "1g.5gb", "uuid": uuid, "memory": 4864, - "appendix": {"vgpu": True, "sliced": True, "mig": True}, + "appendix": {"sliced": True, "mig": True}, } @@ -59,10 +59,10 @@ def test_index_mig_devices_keeps_the_blocks_apart(): assert len(indexes) == len(set(indexes)) -def test_index_mig_devices_clears_physical_indexes(): - # Physical indexes are minor numbers when physical index priority is on, - # so they are not bound by the card count: numbering from the count would - # collide with the cards themselves. +def test_index_mig_devices_offsets_from_the_reported_indexes(): + # A detector's reported index is not necessarily zero-based and contiguous + # -- Ascend reports the DCMI logic id -- so it is not bound by the card + # count: numbering from the count would collide with the cards themselves. cards = [_card(2, "GPU-2"), _card(3, "GPU-3")] mig_devices = { 0: [_mig(0, "MIG-2-0"), _mig(1, "MIG-2-1")], From 4e7f8287c68b8f677403e48b5c60c7744c0f8d5f Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:50:21 +0800 Subject: [PATCH 02/12] fix(detector): report nvidia memory as the driver reports it, and split its usage query - prefer the v2 memory structure through its packed version constant, probed so an older binding stays on v1 without raising the dependency floor - report `memory` as the driver reports it, diverging from the operator's GDDR ECC capacity restore on purpose: the operator's corrected figure is display only and takes no part in allocation, while this one does, so restoring the ~1/16 that ECC parity carves out of a narrow-bus card would make `memory - memory_used` over-report free space and over-commit the card - move GPM, utilization, temperature and used power into `detect_usage`, MIG instance entries included - suppress a failing MIG read per instance rather than per card, so one instance refusing a read no longer aborts the loop and erases every later instance from the inventory, or leaves it reading as idle - delete the vGPU sniff and the `vgpu` appendix key: whole cards only - record `appendix["minor_number"]`, keep the host-memory fallback for a zero total as a deliberate divergence for WSL and iGPU tolerance, and say at the PCI call site why it stays unversioned Signed-off-by: thxCode --- gpustack_runtime/detector/nvidia.py | 465 +++++++---- .../gpustack_runtime/detector/test_nvidia.py | 748 ++++++++++++++++++ 2 files changed, 1077 insertions(+), 136 deletions(-) diff --git a/gpustack_runtime/detector/nvidia.py b/gpustack_runtime/detector/nvidia.py index 4b5be6f..8ec4a51 100644 --- a/gpustack_runtime/detector/nvidia.py +++ b/gpustack_runtime/detector/nvidia.py @@ -18,6 +18,7 @@ ManufacturerEnum, TopologyDistanceEnum, index_mig_devices, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -81,9 +82,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.NVIDIA) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect NVIDIA GPUs using pynvml. + Detect NVIDIA GPUs' inventory using pynvml, without usage metrics. Returns: A list of detected NVIDIA GPU devices, @@ -99,8 +100,6 @@ def detect(self) -> Devices | None: ret: Devices = [] try: - pci_devs = NVIDIADetector.detect_pci_devices() - pynvml.nvmlInit() sys_driver_ver = pynvml.nvmlSystemGetDriverVersion() @@ -134,6 +133,13 @@ def detect(self) -> Devices | None: dev_cc_t = pynvml.nvmlDeviceGetCudaComputeCapability(dev) dev_cc = ".".join(map(str, dev_cc_t)) + # Unversioned on purpose, unlike the memory query above: pynvml + # exposes no v2 PCI accessor to prefer or fall back to -- + # `nvmlDeviceGetPciInfo` is its alias of `nvmlDeviceGetPciInfo_v3`, + # a superset of the v2 structure the operator's `GetPciInfoV` + # prefers, and `nvmlPciInfo_v2_t` is declared but unused. The + # operator never reads the structure's string either, its + # `GetBusId()` formatting the BDF from domain/bus/device. dev_pci_info = pynvml.nvmlDeviceGetPciInfo(dev) dev_bdf = str(dev_pci_info.busIdLegacy).lower() @@ -147,30 +153,27 @@ def detect(self) -> Devices | None: ) dev_numa = bitmask_to_str(list(dev_node_affinity)) - dev_temp = None - with contextlib.suppress(pynvml.NVMLError): - dev_temp = pynvml.nvmlDeviceGetTemperature( - dev, - pynvml.NVML_TEMPERATURE_GPU, - ) - + # The power limit is inventory; the power actually drawn is + # usage, and belongs to detect_usage. dev_power = None - dev_power_used = None with contextlib.suppress(pynvml.NVMLError): dev_power = pynvml.nvmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pynvml.nvmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_mig_mode = pynvml.NVML_DEVICE_MIG_DISABLE with contextlib.suppress(pynvml.NVMLError): dev_mig_mode, _ = pynvml.nvmlDeviceGetMigMode(dev) dev_index = dev_idx - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - with contextlib.suppress(pynvml.NVMLError): - dev_index = pynvml.nvmlDeviceGetMinorNumber(dev) + + # Device.index is the enumeration index, while the driver's + # minor number is what a device node path is made of, so the + # latter goes to the appendix. Mirrors the operator, which + # keeps a sequential Index next to PhysicalIndexes, and omits + # the physical one when the driver cannot answer. + dev_minor_number = None + with contextlib.suppress(pynvml.NVMLError): + dev_minor_number = pynvml.nvmlDeviceGetMinorNumber(dev) # Report the physical card, whether or not MIG is enabled. # MIG instances are partitioned on demand by the operator's @@ -186,52 +189,34 @@ def detect(self) -> Devices | None: with contextlib.suppress(pynvml.NVMLError): dev_cores = pynvml.nvmlDeviceGetNumGpuCores(dev) - dev_cores_util = _get_sm_util_from_gpm_metrics(dev) - if dev_cores_util is None: - with contextlib.suppress(pynvml.NVMLError): - dev_util_rates = pynvml.nvmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pynvml.NVMLError): - dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(dev) - dev_mem = byte_to_mebibyte( # byte to MiB - dev_mem_info.total, - ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( - dev, - pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, - pynvml.NVML_VOLATILE_ECC, - pynvml.NVML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # Reported as the driver reports it, i.e. what the card can + # actually allocate. A deliberate divergence: the operator adds + # back the ~1/16 that ECC parity carves out of a GDDR part, but + # that capacity is not reachable, and its restored figure is a + # display value that takes no part in allocation. Here `memory` + # does take part, and `memory - memory_used` has to mean free + # space, so restoring it would over-commit every GDDR card with + # ECC enabled. + dev_mem, _ = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pynvml.NVML_VOLATILE_ECC, + pynvml.NVML_MEMORY_LOCATION_DRAM, + ) if dev_mem == 0: - dev_mem, dev_mem_used = get_memory() - - dev_is_vgpu = False - if dev_bdf in pci_devs: - dev_is_vgpu = _is_vgpu(pci_devs[dev_bdf].config) + # A deliberate divergence from the operator, which skips a + # device whose total reads 0: here it falls back to the host + # memory, tolerating WSL and integrated GPUs, which report + # no device memory of their own. + dev_mem, _ = get_memory() dev_appendix = { "arch_family": _get_arch_family(dev_cc_t), - "vgpu": dev_is_vgpu, "mig": dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE: dev_mig_slots = 0 with contextlib.suppress(pynvml.NVMLError): @@ -245,9 +230,7 @@ def detect(self) -> Devices | None: sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf, dev_numa, ) @@ -270,14 +253,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -292,6 +270,120 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch NVIDIA GPUs' usage using pynvml, merged into the given devices. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during fetching. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + # The usage query enumerates the driver's devices on its own and returns + # them keyed by UUID, mirroring the operator's MonitorAccelerator: a + # metrics list is joined by device identity, never by index, as an index + # is not stable across a re-detection. + usages: Devices = [] + + try: + pynvml.nvmlInit() + + dev_count = pynvml.nvmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pynvml.nvmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pynvml.nvmlDeviceGetUUID(dev) + + dev_cores_util = _get_sm_util_from_gpm_metrics(dev) + if dev_cores_util is None: + with contextlib.suppress(pynvml.NVMLError): + dev_util_rates = pynvml.nvmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem, dev_mem_used = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pynvml.NVML_VOLATILE_ECC, + pynvml.NVML_MEMORY_LOCATION_DRAM, + ) + if dev_mem == 0: + # The same deliberate divergence detect_info records: a + # device whose total reads 0 falls back to the host memory. + dev_mem, dev_mem_used = get_memory() + + dev_temp = None + with contextlib.suppress(pynvml.NVMLError): + dev_temp = pynvml.nvmlDeviceGetTemperature( + dev, + pynvml.NVML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pynvml.NVMLError): + dev_power_used = ( + pynvml.nvmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + dev_mig_mode = pynvml.NVML_DEVICE_MIG_DISABLE + with contextlib.suppress(pynvml.NVMLError): + dev_mig_mode, _ = pynvml.nvmlDeviceGetMigMode(dev) + if dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE: + dev_mig_slots = 0 + with contextlib.suppress(pynvml.NVMLError): + dev_mig_slots = pynvml.nvmlDeviceGetMaxMigDeviceCount(dev) + usages.extend( + _get_mig_usages( + dev, + dev_mig_slots, + dev_temp, + dev_power_used, + ), + ) + except pynvml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -306,7 +398,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -390,6 +482,86 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_info( + dev: pynvml.c_nvmlDevice_t, +) -> tuple[int, int]: + """ + Get a device's total and used memory, preferring the v2 memory structure + with a v1 fallback, as the operator's `GetMemoryInfoV` does. + + Args: + dev: + The NVML device handle. + + Returns: + The total and used memory in MiB, both 0 if unreadable. + + """ + # `version` is the packed struct version the driver validates, not a plain + # ordinal, so it is the binding's own constant. Probing for it keeps a + # binding predating the v2 structure on the v1 path, instead of raising the + # dependency floor for it. + dev_mem_info_ver = getattr(pynvml, "nvmlMemory_v2", None) + if dev_mem_info_ver is not None: + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo( + dev, + version=dev_mem_info_ver, + ) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(dev) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + return 0, 0 + + +def _get_memory_status( + dev: pynvml.c_nvmlDevice_t, + ecc_counter_type: int, + memory_location: int, +) -> DeviceMemoryStatusEnum: + """ + Get a device's memory health from its uncorrected ECC error counter. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. + + Args: + dev: + The NVML device handle. + ecc_counter_type: + The ECC counter type to read, volatile or aggregate. + memory_location: + The memory location to read the counter of. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pynvml.NVMLError): + dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( + dev, + pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, + ecc_counter_type, + memory_location, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_gpm_metrics( metrics: list[int], dev: pynvml.c_nvmlDevice_t, @@ -603,54 +775,48 @@ def _get_mig_devices( sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf: str, dev_numa, ) -> list[dict]: """ - Enumerate the card's current MIG devices with the same detail a plain - device carries (profile name, uuid, compute/memory utilization, memory - health, temperature and power), returned as appendix entries of the - physical card rather than standalone devices. Empty when MIG is enabled - but no GPU instances exist yet. + Enumerate the card's current MIG devices with the same inventory detail a + plain device carries (profile name, uuid, cores, total memory and memory + health), returned as appendix entries of the physical card rather than + standalone devices. Empty when MIG is enabled but no GPU instances exist + yet. + + An entry keeps a Device's shape, so the fields the usage query owns are + present at a Device's defaults: `_get_mig_usages` fills them. Each entry's `index` is the driver slot the MIG device was found at: index_mig_devices turns it into the device index once every card is detected. """ ret: list[dict] = [] - with contextlib.suppress(pynvml.NVMLError): - for mdev_idx in range(dev_mig_slots): - mdev = None - with contextlib.suppress(pynvml.NVMLError): - mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing a read + # used to abort the loop, so every later instance vanished from the + # inventory. An empty slot raises here as well, which is how it is + # skipped. + with contextlib.suppress(pynvml.NVMLError): + mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) if not mdev: continue mdev_uuid = pynvml.nvmlDeviceGetUUID(mdev) - mdev_mem = 0 - mdev_mem_used = 0 - mdev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pynvml.NVMLError): - mdev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(mdev) - mdev_mem = byte_to_mebibyte(mdev_mem_info.total) - mdev_mem_used = byte_to_mebibyte(mdev_mem_info.used) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - mdev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( - mdev, - pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, - pynvml.NVML_AGGREGATE_ECC, - pynvml.NVML_MEMORY_LOCATION_SRAM, - ) - if mdev_mem_ecc_errors > 0: - mdev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # A MIG device reports the partition's size, which carries no ECC + # reserve to restore: MIG-capable cards are HBM parts. + mdev_mem, _ = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pynvml.NVML_AGGREGATE_ECC, + pynvml.NVML_MEMORY_LOCATION_SRAM, + ) mdev_appendix = { "arch_family": _get_arch_family(dev_cc_t), - "vgpu": True, "sliced": True, "mig": True, "bdf": dev_bdf, @@ -663,8 +829,6 @@ def _get_mig_devices( mdev_ci_id = pynvml.nvmlDeviceGetComputeInstanceId(mdev) mdev_appendix["compute_instance_id"] = mdev_ci_id - mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) - mdev_name = "" mdev_cores = None mdev_gi = pynvml.nvmlDeviceGetGpuInstanceById(dev, mdev_gi_id) @@ -726,20 +890,84 @@ def _get_mig_devices( "runtime_version_original": sys_runtime_ver_original, "compute_capability": dev_cc, "cores": mdev_cores, - "cores_utilization": mdev_cores_util, + "cores_utilization": 0, "memory": mdev_mem, - "memory_used": mdev_mem_used, - "memory_utilization": get_utilization(mdev_mem_used, mdev_mem), + "memory_used": 0, + "memory_utilization": 0, "memory_status": mdev_mem_status, - "temperature": dev_temp, + "temperature": None, "power": dev_power, - "power_used": dev_power_used, + "power_used": None, "appendix": mdev_appendix, }, ) return ret +def _get_mig_usages( + dev, + dev_mig_slots: int, + dev_temp, + dev_power_used, +) -> Devices: + """ + Fetch the usage of the card's current MIG devices, one UUID-keyed entry per + instance, to merge into the card's `appendix["mig_devices"]`. + + Args: + dev: + The NVML device handle of the card hosting them. + dev_mig_slots: + The number of MIG devices the card can host. + dev_temp: + The card's temperature. + dev_power_used: + The card's used power. + + Returns: + The MIG devices' usage, keyed by UUID. + + """ + ret: Devices = [] + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing its UUID + # or its GPU instance id used to abort the loop, so every later instance + # kept the inventory's defaults -- 0 % and 0 MiB, reported idle while it + # may be running a workload. An empty slot raises here as well, which is + # how it is skipped. + with contextlib.suppress(pynvml.NVMLError): + mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + if not mdev: + continue + + mdev_uuid = pynvml.nvmlDeviceGetUUID(mdev) + + mdev_mem, mdev_mem_used = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pynvml.NVML_AGGREGATE_ECC, + pynvml.NVML_MEMORY_LOCATION_SRAM, + ) + + mdev_gi_id = pynvml.nvmlDeviceGetGpuInstanceId(mdev) + mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) + + ret.append( + Device( + uuid=mdev_uuid, + cores_utilization=mdev_cores_util, + memory_used=mdev_mem_used, + memory_utilization=get_utilization(mdev_mem_used, mdev_mem), + memory_status=mdev_mem_status, + # A MIG device reports neither temperature nor power, so it + # carries the card's. + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + return ret + + def _get_arch_family(dev_cc_t: list[int]) -> str: """ Get the architecture family based on the CUDA compute capability. @@ -774,38 +1002,3 @@ def _get_arch_family(dev_cc_t: list[int]) -> str: case 10 | 12: return "Blackwell" return "Unknown" - - -def _is_vgpu(dev_config: bytes) -> bool: - """ - Determine if the device is a vGPU based on its PCI configuration space. - - """ - status = 0x06 - cap_supported = 0x10 - cap_start = 0x34 - cap_vendor_specific_id = 0x09 - - if dev_config[status] & cap_supported == 0: - return False - - # Find the capability list - dev_cap: bytes | None = None - visited = set() - pos = dev_config[cap_start] - while pos != 0 and pos not in visited and pos < len(dev_config) - 2: - visited.add(pos) - ptr = dev_config[pos : pos + 3] # id, next, length - if ptr[0] == 0xFF: - break - if ptr[0] == cap_vendor_specific_id: - dev_cap = dev_config[pos : pos + ptr[2]] - break - pos = ptr[1] - - if not dev_cap or len(dev_cap) < 5: - return False - - # Check for vGPU signature, - # which is either 0x56 (NVIDIA vGPU) or 0x46 (NVIDIA GRID). - return dev_cap[3] == 0x56 or dev_cap[4] == 0x46 diff --git a/tests/gpustack_runtime/detector/test_nvidia.py b/tests/gpustack_runtime/detector/test_nvidia.py index b128212..1562e34 100644 --- a/tests/gpustack_runtime/detector/test_nvidia.py +++ b/tests/gpustack_runtime/detector/test_nvidia.py @@ -1,7 +1,755 @@ +from __future__ import annotations + +import ctypes +import dataclasses +from types import SimpleNamespace + import pytest +from gpustack_runtime.detector import nvidia +from gpustack_runtime.detector.__types__ import DeviceMemoryStatusEnum +from gpustack_runtime.detector.__utils__ import get_utilization from gpustack_runtime.detector.nvidia import NVIDIADetector +# --------------------------------------------------------------------------- # +# A fake pynvml, built in this module rather than shared: every vendor binding # +# exposes an unrelated API, and this one carries a call log because several # +# criteria are "issues no metric call", which no return value can prove. # +# --------------------------------------------------------------------------- # + +_MIB = 1 << 20 + +_USAGE_CALLS = ( + "nvmlGpmQueryDeviceSupport", + "nvmlGpmSampleGet", + "nvmlGpmMigSampleGet", + "nvmlGpmMetricsGet", + "nvmlDeviceGetUtilizationRates", + "nvmlDeviceGetTemperature", + "nvmlDeviceGetPowerUsage", +) +""" +The driver calls only the usage query is allowed to make. +""" + + +class _NVMLError(Exception): + """ + The fake binding's error type, standing in for pynvml.NVMLError. + """ + + +class _FakeFabricInfo(ctypes.Structure): + """ + A byref()-able fabric info: the detector takes this struct's address. + """ + + _fields_ = ( + ("state", ctypes.c_uint), + ("clusterUuid", ctypes.c_ubyte * 16), + ("cliqueId", ctypes.c_uint), + ) + + +class _FakeGpmMetrics: + """ + A GPM metrics request, as the detector fills and reads it. + """ + + def __init__(self): + self.version = 0 + self.numMetrics = 0 + self.sample1 = None + self.sample2 = None + self.metrics = [SimpleNamespace(metricId=0, value=float("nan"))] + + +@dataclasses.dataclass +class _FakeMigDevice: + """ + A MIG device of a MIG-enabled card. + """ + + uuid: str + gpu_instance_id: int = 0 + compute_instance_id: int = 0 + memory: int = 4864 * _MIB + memory_used: int = 512 * _MIB + ecc_errors: int = 0 + sm_util: float | None = None + mig_devices: None = None + + +@dataclasses.dataclass +class _FakeDevice: + """ + A card, as the fake binding reports it. Every field is a knob a test turns. + """ + + uuid: str = "GPU-0" + name: str = "NVIDIA L4" + minor_number: int | None = 0 + bdf: str = "0000:6A:00.0" + compute_capability: tuple[int, int] = (8, 9) + cores: int = 7424 + memory: int = 23034 * _MIB + memory_used: int = 1024 * _MIB + memory_bus_width: int | None = 192 + ecc_mode: int = 1 # NVML_FEATURE_ENABLED + ecc_errors: int = 0 + temperature: int = 47 + power_limit: int = 72_000 # mW + power_used: int = 30_000 # mW + cores_utilization: int = 12 + sm_util: float | None = None + """ + The SM utilization GPM samples, or None for a card not supporting GPM. + """ + mig_devices: list[_FakeMigDevice] | None = None + """ + The card's MIG devices, or None for a card with MIG disabled. + """ + + +class _FakeNVML: + """ + A pynvml stand-in exposing only what the NVIDIA detector touches, recording + every call it receives. + """ + + NVMLError = _NVMLError + NVML_SUCCESS = 0 + NVML_ERROR_NOT_SUPPORTED = 3 + NVML_FEATURE_DISABLED = 0 + NVML_FEATURE_ENABLED = 1 + NVML_TEMPERATURE_GPU = 0 + NVML_AFFINITY_SCOPE_NODE = 1 + NVML_DEVICE_MIG_DISABLE = 0 + NVML_DEVICE_MIG_ENABLE = 1 + NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1 + NVML_VOLATILE_ECC = 0 + NVML_AGGREGATE_ECC = 1 + NVML_MEMORY_LOCATION_DRAM = 2 + NVML_MEMORY_LOCATION_SRAM = 5 + NVML_GPU_INSTANCE_PROFILE_COUNT = 3 + NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 2 + NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 1 + NVML_GPM_METRIC_SM_UTIL = 2 + NVML_GPM_METRICS_GET_VERSION = 1 + NVML_GPU_FABRIC_STATE_COMPLETED = 3 + + c_nvmlGpuFabricInfoV_t = _FakeFabricInfo # noqa: N815 + c_nvmlGpmMetricsGet_t = _FakeGpmMetrics # noqa: N815 + + mig_profile_name = "MIG 1g.5gb" + mig_profile_memory_mb = 4864 + mig_profile_cores = 14 + + def __init__( + self, + devices: list[_FakeDevice], + memory_v2_binding: bool = True, + memory_v2_driver: bool = True, + ): + """ + Args: + devices: + The cards the fake driver enumerates. + memory_v2_binding: + Whether the binding exposes the packed v2 struct version, as an + older nvidia-ml-py does not. + memory_v2_driver: + Whether the driver answers the v2 memory call. + + """ + self.devices = list(devices) + self.calls: list[str] = [] + self.memory_versions: list[int | None] = [] + self.memory_v2_driver = memory_v2_driver + if memory_v2_binding: + self.nvmlMemory_v2 = 33554472 + self._gpm_target: _FakeDevice | _FakeMigDevice | None = None + + # System. + + def nvmlInit(self): # noqa: N802 + self.calls.append("nvmlInit") + + def nvmlSystemGetDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetDriverVersion") + return "580.65.06" + + def nvmlSystemGetCudaDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetCudaDriverVersion") + return 13000 + + def nvmlDeviceGetCount(self): # noqa: N802 + self.calls.append("nvmlDeviceGetCount") + return len(self.devices) + + def nvmlDeviceGetHandleByIndex(self, index): # noqa: N802 + self.calls.append("nvmlDeviceGetHandleByIndex") + return self.devices[index] + + # Identity and capability. + + def nvmlDeviceGetUUID(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetUUID") + return handle.uuid + + def nvmlDeviceGetName(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetName") + return handle.name + + def nvmlDeviceGetCudaComputeCapability(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetCudaComputeCapability") + return list(handle.compute_capability) + + def nvmlDeviceGetPciInfo(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPciInfo") + # The legacy tuple is uppercase hexadecimal, as NVML formats it. + return SimpleNamespace(busIdLegacy=handle.bdf) + + def nvmlDeviceGetMemoryAffinity(self, handle, size, scope): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryAffinity") + msg = "no memory affinity" + raise _NVMLError(msg) + + def nvmlDeviceGetNumGpuCores(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetNumGpuCores") + return handle.cores + + def nvmlDeviceGetMinorNumber(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMinorNumber") + if handle.minor_number is None: + msg = "no minor number" + raise _NVMLError(msg) + return handle.minor_number + + def nvmlDeviceGetPowerManagementDefaultLimit(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerManagementDefaultLimit") + return handle.power_limit + + def nvmlDeviceGetGpuFabricInfoV(self, handle, info_ref): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuFabricInfoV") + return self.NVML_ERROR_NOT_SUPPORTED + + # Memory. + + def nvmlDeviceGetMemoryInfo(self, handle, version=None): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryInfo") + self.memory_versions.append(version) + if version is not None and not self.memory_v2_driver: + msg = "no v2 memory info" + raise _NVMLError(msg) + return SimpleNamespace( + total=handle.memory, + used=handle.memory_used, + free=handle.memory - handle.memory_used, + ) + + def nvmlDeviceGetMemoryBusWidth(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryBusWidth") + if handle.memory_bus_width is None: + msg = "no memory bus width" + raise _NVMLError(msg) + return handle.memory_bus_width + + def nvmlDeviceGetEccMode(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetEccMode") + return [handle.ecc_mode, handle.ecc_mode] + + def nvmlDeviceGetMemoryErrorCounter(self, handle, error_type, scope, location): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryErrorCounter") + return handle.ecc_errors + + # Usage. + + def nvmlDeviceGetUtilizationRates(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetUtilizationRates") + return SimpleNamespace(gpu=handle.cores_utilization, memory=0) + + def nvmlDeviceGetTemperature(self, handle, sensor): # noqa: N802 + self.calls.append("nvmlDeviceGetTemperature") + return handle.temperature + + def nvmlDeviceGetPowerUsage(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerUsage") + return handle.power_used + + def nvmlGpmQueryDeviceSupport(self, handle): # noqa: N802 + self.calls.append("nvmlGpmQueryDeviceSupport") + return SimpleNamespace(isSupportedDevice=int(handle.sm_util is not None)) + + def nvmlGpmSampleAlloc(self): # noqa: N802 + self.calls.append("nvmlGpmSampleAlloc") + return object() + + def nvmlGpmSampleFree(self, sample): # noqa: N802 + self.calls.append("nvmlGpmSampleFree") + + def nvmlGpmSampleGet(self, handle, sample): # noqa: N802 + self.calls.append("nvmlGpmSampleGet") + self._gpm_target = handle + + def nvmlGpmMigSampleGet(self, handle, gpu_instance_id, sample): # noqa: N802 + self.calls.append("nvmlGpmMigSampleGet") + self._gpm_target = next( + ( + mig + for mig in handle.mig_devices or [] + if mig.gpu_instance_id == gpu_instance_id + ), + None, + ) + + def nvmlGpmMetricsGet(self, metrics): # noqa: N802 + self.calls.append("nvmlGpmMetricsGet") + sm_util = getattr(self._gpm_target, "sm_util", None) + metrics.metrics[0].value = float("nan") if sm_util is None else float(sm_util) + + # MIG. + + def nvmlDeviceGetMigMode(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMigMode") + mode = ( + self.NVML_DEVICE_MIG_DISABLE + if handle.mig_devices is None + else self.NVML_DEVICE_MIG_ENABLE + ) + return mode, mode + + def nvmlDeviceGetMaxMigDeviceCount(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMaxMigDeviceCount") + return 7 + + def nvmlDeviceGetMigDeviceHandleByIndex(self, handle, index): # noqa: N802 + self.calls.append("nvmlDeviceGetMigDeviceHandleByIndex") + migs = handle.mig_devices or [] + if index >= len(migs): + msg = "no MIG device at that slot" + raise _NVMLError(msg) + return migs[index] + + def nvmlDeviceGetGpuInstanceId(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceId") + return handle.gpu_instance_id + + def nvmlDeviceGetComputeInstanceId(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetComputeInstanceId") + return handle.compute_instance_id + + def nvmlDeviceGetGpuInstanceById(self, handle, gpu_instance_id): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceById") + return SimpleNamespace(card=handle, gpu_instance_id=gpu_instance_id) + + def nvmlGpuInstanceGetComputeInstanceById(self, gpu_instance, compute_instance_id): # noqa: N802 + self.calls.append("nvmlGpuInstanceGetComputeInstanceById") + return SimpleNamespace(compute_instance_id=compute_instance_id) + + def nvmlGpuInstanceGetInfo(self, gpu_instance): # noqa: N802 + self.calls.append("nvmlGpuInstanceGetInfo") + return SimpleNamespace(profileId=0) + + def nvmlComputeInstanceGetInfo(self, compute_instance): # noqa: N802 + self.calls.append("nvmlComputeInstanceGetInfo") + return SimpleNamespace(profileId=0) + + def nvmlDeviceGetGpuInstanceProfileInfo(self, handle, profile_id): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceProfileInfo") + if profile_id != 0: + msg = "no such GPU instance profile" + raise _NVMLError(msg) + return SimpleNamespace( + id=0, + memorySizeMB=self.mig_profile_memory_mb, + sliceCount=1, + name=self.mig_profile_name, + ) + + def nvmlGpuInstanceGetComputeInstanceProfileInfo( # noqa: N802 + self, + gpu_instance, + profile_id, + engine_profile_id, + ): + self.calls.append("nvmlGpuInstanceGetComputeInstanceProfileInfo") + if (profile_id, engine_profile_id) != (0, 0): + msg = "no such compute instance profile" + raise _NVMLError(msg) + return SimpleNamespace(id=0, multiprocessorCount=self.mig_profile_cores) + + +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the ECC error check on: it is opt-in, as reading the counters costs a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults + to true. A real module attribute is set because the env lookup is cached. + """ + monkeypatch.setattr( + nvidia.envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + +@pytest.fixture +def fake_nvml(monkeypatch): + """ + Install a fake binding in place of pynvml, and return the installer. + """ + + def _install(devices: list[_FakeDevice], **kwargs) -> _FakeNVML: + fake = _FakeNVML(devices, **kwargs) + monkeypatch.setattr(nvidia, "pynvml", fake) + # is_supported() initializes the real driver, which the fake replaces. + monkeypatch.setattr(NVIDIADetector, "is_supported", staticmethod(lambda: True)) + # The NUMA node comes from sysfs, so it is answered here instead of + # letting the host decide what the test sees. + monkeypatch.setattr(nvidia, "get_numa_node_by_bdf", lambda *_: "") + # GPM samples over a 100 ms window of real time, twice per query. + monkeypatch.setattr(nvidia.time, "sleep", lambda *_: None) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# The information query. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_the_card(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect_info() + + assert len(devices) == 1 + device = devices[0] + assert device.index == 0 + assert device.name == "NVIDIA L4" + assert device.uuid == "GPU-0" + assert device.driver_version == "580.65.06" + assert device.runtime_version == "13.0" + assert device.runtime_version_original == "13.0.0" + assert device.compute_capability == "8.9" + assert device.cores == 7424 + assert device.power == 72 + assert device.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert device.appendix == { + "arch_family": "Ada-Lovelace", + "mig": False, + "bdf": "0000:6a:00.0", + "minor_number": 0, + } + # The usage fields stay at their defaults: this query does not read them. + assert device.cores_utilization == 0 + assert device.memory_used == 0 + assert device.memory_utilization == 0 + assert device.temperature is None + assert device.power_used is None + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + + +def test_detect_info_issues_no_usage_call(fake_nvml): + # A card supporting GPM and hosting MIG devices, i.e. every usage call the + # detector knows is reachable. + fake = fake_nvml( + [ + _FakeDevice( + uuid="GPU-0", + name="NVIDIA H100 80GB HBM3", + memory_bus_width=5120, + sm_util=61.0, + mig_devices=[_FakeMigDevice(uuid="MIG-0-0", sm_util=33.0)], + ), + ], + ) + + NVIDIADetector().detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # ... while the inventory calls did happen. + assert "nvmlDeviceGetMemoryInfo" in fake.calls + assert "nvmlDeviceGetPowerManagementDefaultLimit" in fake.calls + assert "nvmlDeviceGetMigDeviceHandleByIndex" in fake.calls + + +def test_detect_info_records_no_vgpu(fake_nvml): + # Whole-card reporting: no virtual/PF/VF classification anywhere. + fake_nvml([_FakeDevice(mig_devices=[_FakeMigDevice(uuid="MIG-0-0")])]) + + devices = NVIDIADetector().detect_info() + + assert "vgpu" not in devices[0].appendix + assert "vgpu" not in devices[0].appendix["mig_devices"][0]["appendix"] + assert not hasattr(nvidia, "_is_vgpu") + + +def test_detect_info_omits_an_unreadable_minor_number(fake_nvml): + fake_nvml([_FakeDevice(minor_number=None)]) + + devices = NVIDIADetector().detect_info() + + assert "minor_number" not in devices[0].appendix + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_reports_the_memory_health(fake_nvml): + fake_nvml([_FakeDevice(ecc_errors=1)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# Memory is what the card can allocate, not its ECC-restored capacity. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "bus_width, ecc_mode", + [ + (192, 1), # L4: GDDR with ECC on -- where the operator would restore. + (384, 1), # L40S: the widest GDDR bus. + (5120, 1), # H100: HBM keeps ECC out of the user-visible memory anyway. + (384, 0), # GDDR, ECC off: nothing was carved out. + (None, 1), # An unreadable bus width. + ], +) +def test_detect_info_reports_the_allocatable_memory(fake_nvml, bus_width, ecc_mode): + # A deliberate divergence from the operator, which adds back the ~1/16 that + # ECC parity carves out of a GDDR part. That capacity is not reachable, and + # the operator's restored figure is a display value taking no part in + # allocation -- but `memory` here does, and `memory - memory_used` has to + # mean free space, so restoring it would over-commit the card. + total = 23034 + fake_nvml( + [ + _FakeDevice( + memory=total * _MIB, + memory_bus_width=bus_width, + ecc_mode=ecc_mode, + ), + ], + ) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory == total + + +def test_detect_falls_back_to_the_host_memory(fake_nvml, monkeypatch): + # A deliberate divergence from the operator, which skips such a device. + fake_nvml([_FakeDevice(memory=0, memory_used=0)]) + monkeypatch.setattr(nvidia, "get_memory", lambda: (65536, 4096)) + + detector = NVIDIADetector() + devices = detector.detect_info() + + assert devices[0].memory == 65536 + assert devices[0].memory_used == 0 + + detector.detect_usage(devices) + + assert devices[0].memory_used == 4096 + assert devices[0].memory_utilization == get_utilization(4096, 65536) + + +# --------------------------------------------------------------------------- # +# The v2 memory structure. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_v2_memory_structure(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + NVIDIADetector().detect_info() + + # The packed struct version the driver validates, not a plain ordinal. + assert fake.memory_versions == [fake.nvmlMemory_v2] + + +def test_detect_info_falls_back_to_the_v1_memory_structure(fake_nvml): + fake = fake_nvml([_FakeDevice()], memory_v2_driver=False) + + devices = NVIDIADetector().detect_info() + + assert fake.memory_versions == [fake.nvmlMemory_v2, None] + assert devices[0].memory == 23034 + + +def test_detect_info_skips_the_v2_call_the_binding_lacks(fake_nvml): + fake = fake_nvml([_FakeDevice()], memory_v2_binding=False) + + devices = NVIDIADetector().detect_info() + + assert fake.memory_versions == [None] + assert devices[0].memory == 23034 + + +# --------------------------------------------------------------------------- # +# The usage query. # +# --------------------------------------------------------------------------- # + + +def test_detect_fills_the_usage_fields(fake_nvml): + fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect() + + device = devices[0] + assert device.cores_utilization == 12 + assert device.memory_used == 1024 + # The three memory fields of one card agree: the utilization is measured + # against the same total that `memory` reports. + assert device.memory == 23034 + assert device.memory_utilization == get_utilization(1024, 23034) + assert device.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert device.temperature == 47 + assert device.power_used == 30 + + +def test_detect_without_usage_leaves_the_usage_fields_alone(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect(usage=False) + + assert devices[0].cores_utilization == 0 + assert devices[0].temperature is None + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + + +def test_detect_usage_merges_into_the_given_devices(fake_nvml): + fake_nvml([_FakeDevice(uuid="GPU-0"), _FakeDevice(uuid="GPU-1")]) + detector = NVIDIADetector() + devices = detector.detect_info() + + merged = detector.detect_usage(devices) + + assert merged is devices + assert [device.temperature for device in devices] == [47, 47] + # The information fields survive the merge. + assert [device.uuid for device in devices] == ["GPU-0", "GPU-1"] + assert [device.name for device in devices] == ["NVIDIA L4"] * 2 + assert [device.memory for device in devices] == [23034] * 2 + + +def test_detect_usage_detects_the_information_first(fake_nvml): + fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect_usage() + + assert [device.uuid for device in devices] == ["GPU-0"] + assert devices[0].cores_utilization == 12 + + +def test_detect_usage_prefers_gpm(fake_nvml): + fake = fake_nvml([_FakeDevice(sm_util=61.4)]) + + devices = NVIDIADetector().detect() + + assert devices[0].cores_utilization == 61 + assert "nvmlDeviceGetUtilizationRates" not in fake.calls + + +@pytest.mark.usefixtures("health_check") +def test_detect_usage_reports_the_memory_health(fake_nvml): + fake_nvml([_FakeDevice(ecc_errors=1)]) + + devices = NVIDIADetector().detect() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# The MIG devices, which stay in the card's appendix. # +# --------------------------------------------------------------------------- # + + +def _mig_card(**kwargs) -> _FakeDevice: + return _FakeDevice( + uuid="GPU-0", + name="NVIDIA A100-SXM4-40GB", + compute_capability=(8, 0), + memory=40960 * _MIB, + memory_bus_width=5120, + # GPM is queried on the card, whether the sample is the card's or an + # instance's. + sm_util=55.0, + mig_devices=[ + _FakeMigDevice(uuid="MIG-0-0", gpu_instance_id=1, sm_util=33.0), + _FakeMigDevice(uuid="MIG-0-1", gpu_instance_id=2, sm_util=44.0), + ], + **kwargs, + ) + + +def test_detect_info_keeps_the_mig_devices_in_the_appendix(fake_nvml): + fake_nvml([_mig_card()]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].appendix["mig"] is True + mig_devices = devices[0].appendix["mig_devices"] + assert [mig["uuid"] for mig in mig_devices] == ["MIG-0-0", "MIG-0-1"] + assert [mig["name"] for mig in mig_devices] == ["1g.5gb"] * 2 + # Numbered by index_mig_devices: a block above the cards' own indexes. + assert [mig["index"] for mig in mig_devices] == [1, 2] + assert [mig["memory"] for mig in mig_devices] == [4864] * 2 + assert [mig["cores"] for mig in mig_devices] == [14] * 2 + assert mig_devices[0]["appendix"]["sliced"] is True + assert mig_devices[0]["appendix"]["mig"] is True + assert mig_devices[0]["appendix"]["gpu_instance_id"] == 1 + # The usage fields of an instance stay at their defaults too. + assert mig_devices[0]["cores_utilization"] == 0 + assert mig_devices[0]["memory_used"] == 0 + assert mig_devices[0]["memory_utilization"] == 0 + assert mig_devices[0]["temperature"] is None + assert mig_devices[0]["power_used"] is None + + +def test_detect_fills_the_mig_devices_usage(fake_nvml): + fake_nvml([_mig_card()]) + + devices = NVIDIADetector().detect() + + mig_devices = devices[0].appendix["mig_devices"] + assert [mig["cores_utilization"] for mig in mig_devices] == [33, 44] + assert [mig["memory_used"] for mig in mig_devices] == [512] * 2 + assert [mig["memory_utilization"] for mig in mig_devices] == [ + get_utilization(512, 4864), + ] * 2 + assert [mig["memory_status"] for mig in mig_devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + ] * 2 + # An instance reports neither temperature nor power, so it carries the + # card's, as the information query's entries do. + assert [mig["temperature"] for mig in mig_devices] == [47] * 2 + assert [mig["power_used"] for mig in mig_devices] == [30] * 2 + # The instances' own information fields are untouched by the merge. + assert [mig["index"] for mig in mig_devices] == [1, 2] + assert [mig["memory"] for mig in mig_devices] == [4864] * 2 + + +def test_detect_reports_a_mig_enabled_card_without_instances(fake_nvml): + # MIG on, nothing partitioned yet: the card is still the reported device. + fake_nvml([_FakeDevice(mig_devices=[])]) + + devices = NVIDIADetector().detect() + + assert len(devices) == 1 + assert devices[0].appendix["mig"] is True + assert devices[0].appendix["mig_devices"] == [] + + +# --------------------------------------------------------------------------- # +# The hardware paths, exercised only where a driver exists. # +# --------------------------------------------------------------------------- # + @pytest.mark.skipif( not NVIDIADetector.is_supported(), From 917492503690a0b0c58a70f6ac58da84a3f4f14c Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:50:31 +0800 Subject: [PATCH 03/12] fix(detector): detect ascend npus on an older driver, and only npus - skip a unit whose DCMI type is not NPU, keeping one whose type cannot be read, as the operator does - bind the V1 die, PCIe and memory entry points and fall back to them, so a driver exposing only those still yields a device instead of none, pinning both new structs' size and field offsets against the driver's header - derive used memory from the utilization percent on the V2 memory path, rather than reporting every card as fully used the way the operator does there - drop the vNPU branch and the `vgpu` appendix key, and with them the `/dev/vdavinci` device node path - move utilization, temperature and power into `detect_usage` - drop a device whose physical id DCMI will not answer, as the operator's ascend/device.go does on a failed GetPhysicalID: /dev/davinciN is numbered by that id, so such a device cannot be addressed at all - stop the CDI generator standing `Device.index` in for a missing physical id -- the index is the logic id, a different number, so the fallback could hand a container another NPU's node. The physical id is now an invariant of a detected Ascend device. Signed-off-by: thxCode --- gpustack_runtime/deployer/cdi/ascend.py | 14 +- gpustack_runtime/detector/ascend.py | 418 +++++++--- gpustack_runtime/detector/pydcmi/__init__.py | 52 ++ .../gpustack_runtime/detector/test_ascend.py | 712 ++++++++++++++++++ 4 files changed, 1073 insertions(+), 123 deletions(-) diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index fa5e88e..40b1ac1 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -112,9 +112,17 @@ def generate( container_device_nodes = [] - cdn_path = f"/dev/davinci{dev.index}" - if dev.appendix.get("vgpu", False): - cdn_path = f"/dev/vdavinci{dev.index}" + # The device node is numbered by the driver's physical id, which + # Device.index no longer carries: it is the detector's enumeration + # index, i.e. the DCMI logic id here. The two are different + # numbers, so a device without a physical id is skipped instead of + # addressed by the index, which would resolve to another NPU's + # node. The detector already drops such a device; this guards the + # devices a caller passes in. + cdn_number = dev.appendix.get("physical_id") + if cdn_number is None: + continue + cdn_path = f"/dev/davinci{cdn_number}" cdn = device_to_cdi_device_node( path=cdn_path, ) diff --git a/gpustack_runtime/detector/ascend.py b/gpustack_runtime/detector/ascend.py index a88fe31..0f6208f 100644 --- a/gpustack_runtime/detector/ascend.py +++ b/gpustack_runtime/detector/ascend.py @@ -18,6 +18,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -92,9 +93,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.ASCEND) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Ascend NPUs using pydcmi. + Detect Ascend NPUs' inventory using pydcmi, without usage metrics. Returns: A list of detected Ascend NPU devices, @@ -121,82 +122,54 @@ def detect(self) -> Devices | None: for dev_card_id in card_list: device_num_in_card = pydcmi.dcmi_get_device_num_in_card(dev_card_id) for dev_device_id in range(device_num_in_card): - dev_is_vgpu = False - dev_virt_info = _get_device_virtual_info( + if not _is_npu_device(dev_card_id, dev_device_id): + continue + + dev_chip_info = _get_device_chip_info( dev_card_id, dev_device_id, ) - if ( - dev_virt_info - and hasattr(dev_virt_info, "query_info") - and hasattr(dev_virt_info.query_info, "computing") - ): - dev_is_vgpu = True - dev_cores_aicore = dev_virt_info.query_info.computing.aic - dev_name = dev_virt_info.query_info.name - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - if hasattr(dev_virt_info.query_info.computing, "memory_size"): - dev_mem = dev_virt_info.query_info.computing.memory_size - dev_index = dev_virt_info.vdev_id - else: - dev_chip_info = pydcmi.dcmi_get_device_chip_info_v2( - dev_card_id, - dev_device_id, - ) - dev_cores_aicore = dev_chip_info.aicore_cnt - dev_name = dev_chip_info.chip_name - dev_mem, dev_mem_used = _get_device_memory_info( - dev_card_id, - dev_device_id, - ) - dev_mem_status = _get_device_memory_status( - dev_card_id, - dev_device_id, - ) - dev_index = pydcmi.dcmi_get_device_logic_id( - dev_card_id, - dev_device_id, - ) - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - dev_index = pydcmi.dcmi_get_device_phyid_from_logicid( - dev_index, - ) - dev_uuid = pydcmi.dcmi_get_device_die_v2( + dev_cores_aicore = dev_chip_info.aicore_cnt + dev_name = dev_chip_info.chip_name + dev_mem, _ = _get_device_memory_info( dev_card_id, dev_device_id, - pydcmi.DCMI_DIE_TYPE_VDIE, ) - - dev_util_aicore = pydcmi.dcmi_get_device_utilization_rate( + dev_mem_status = _get_device_memory_status( dev_card_id, dev_device_id, - pydcmi.DCMI_INPUT_TYPE_AICORE, ) - if dev_util_aicore is None: + dev_index = pydcmi.dcmi_get_device_logic_id( + dev_card_id, + dev_device_id, + ) + # Device.index is the logic id the driver enumerates + # the NPU at, while the physical id is what a device + # node path is made of, so the latter goes to the + # appendix beside the card and device ids. Mirrors the + # operator, which keeps a sequential Index next to + # PhysicalIndexes. + # + # A device whose physical id cannot be read cannot be + # addressed at all: /dev/davinciN is numbered by it, and + # the logic id is a different number, so standing in for + # it would hand a container another NPU's node. The + # operator skips such a device for the same reason. + try: + dev_physical_id = pydcmi.dcmi_get_device_phyid_from_logicid( + dev_index, + ) + except pydcmi.DCMIError: debug_log_warning( logger, - "Failed to get device %d cores utilization, setting to 0", + "Failed to fetch physical id of device %d, skipping it", dev_index, ) - dev_util_aicore = 0 - - dev_temp = pydcmi.dcmi_get_device_temperature( - dev_card_id, - dev_device_id, - ) + continue - dev_power_used = None - with contextlib.suppress(pydcmi.DCMIError): - dev_power_used = pydcmi.dcmi_get_device_power_info( - dev_card_id, - dev_device_id, - ) - if dev_power_used: - dev_power_used = dev_power_used / 10 # 0.1W to W + dev_uuid = _get_device_die(dev_card_id, dev_device_id) - dev_bdf = pydcmi.dcmi_get_device_bdf( + dev_bdf = _get_device_bdf( dev_card_id, dev_device_id, ) @@ -214,11 +187,11 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _guess_soc_name_from_dev_name(dev_name), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, "card_id": dev_card_id, "device_id": dev_device_id, "device_id_max": device_num_in_card - 1, + "physical_id": dev_physical_id, } if dev_numa: dev_appendix["numa"] = dev_numa @@ -246,13 +219,8 @@ def detect(self) -> Devices | None: runtime_version=sys_runtime_ver, runtime_version_original=sys_runtime_ver_original, cores=dev_cores_aicore, - cores_utilization=dev_util_aicore, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -265,6 +233,110 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Ascend NPUs' usage using pydcmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pydcmi.dcmi_init() + + _, card_list = pydcmi.dcmi_get_card_list() + for dev_card_id in card_list: + device_num_in_card = pydcmi.dcmi_get_device_num_in_card(dev_card_id) + for dev_device_id in range(device_num_in_card): + # The operator filters by device type in MonitorAccelerator + # as well, not only when detecting. + if not _is_npu_device(dev_card_id, dev_device_id): + continue + + dev_uuid = _get_device_die(dev_card_id, dev_device_id) + + # The operator's MonitorAccelerator re-reads the memory + # rather than trusting the detection pass. + dev_mem, dev_mem_used = _get_device_memory_info( + dev_card_id, + dev_device_id, + ) + dev_mem_status = _get_device_memory_status( + dev_card_id, + dev_device_id, + ) + + dev_util_aicore = None + with contextlib.suppress(pydcmi.DCMIError): + dev_util_aicore = pydcmi.dcmi_get_device_utilization_rate( + dev_card_id, + dev_device_id, + pydcmi.DCMI_INPUT_TYPE_AICORE, + ) + if dev_util_aicore is None: + debug_log_warning( + logger, + "Failed to get device %d/%d cores utilization, " + "setting to 0", + dev_card_id, + dev_device_id, + ) + dev_util_aicore = 0 + + dev_temp = None + with contextlib.suppress(pydcmi.DCMIError): + dev_temp = pydcmi.dcmi_get_device_temperature( + dev_card_id, + dev_device_id, + ) + + dev_power_used = None + with contextlib.suppress(pydcmi.DCMIError): + dev_power_used = pydcmi.dcmi_get_device_power_info( + dev_card_id, + dev_device_id, + ) + if dev_power_used: + dev_power_used = dev_power_used / 10 # 0.1W to W + + usages.append( + Device( + uuid=dev_uuid.upper(), + cores_utilization=dev_util_aicore, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pydcmi.DCMIError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between Ascend NPUs. @@ -279,7 +351,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -340,6 +412,118 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _is_npu_device(dev_card_id, dev_device_id) -> bool: + """ + Report whether the given device of the card is an NPU. + + A card also carries non-NPU units, like its MCU, which are not + accelerators. Mirrors the operator, which skips a device only when the + type call *succeeds* and reports something other than an NPU: a device + whose type cannot be read is kept. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + True if the device is an NPU, or its type is unreadable. + + """ + dev_type = None + with contextlib.suppress(pydcmi.DCMIError): + dev_type = pydcmi.dcmi_get_device_type(dev_card_id, dev_device_id) + + if dev_type is not None and dev_type != pydcmi.DCMI_UNIT_TYPE_NPU: + slogger.debug( + "Skipping non-NPU device %d of card %d, type %d", + dev_device_id, + dev_card_id, + dev_type, + ) + return False + + return True + + +def _get_device_die(dev_card_id, dev_device_id) -> str: + """ + Get the device's SoC die, which identifies it. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The die as a string. + + """ + try: + return pydcmi.dcmi_get_device_die_v2( + dev_card_id, + dev_device_id, + pydcmi.DCMI_DIE_TYPE_VDIE, + ) + except pydcmi.DCMIError: + # An older driver exposes the V1 call only, which takes no die type + # and reports the SoC die directly. Mirrors the operator's + # VDieHandler, which tries V2 then V1. + return pydcmi.dcmi_get_device_die(dev_card_id, dev_device_id) + + +def _get_device_bdf(dev_card_id, dev_device_id) -> str: + """ + Get the device's PCI bus address. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The BDF as a string. + + """ + try: + return pydcmi.dcmi_get_device_bdf(dev_card_id, dev_device_id) + except pydcmi.DCMIError: + # An older driver exposes the V1 PCIe call only, whose struct carries + # no PCI domain, so the domain reads as 0 -- as the operator's + # PcieInfoHandler.V1 leaves it when widening to the V2 struct. + dev_pcie_info = pydcmi.dcmi_get_device_pcie_info(dev_card_id, dev_device_id) + return ( + f"0000:{dev_pcie_info.bdf_busid:02x}:" + f"{dev_pcie_info.bdf_deviceid:02x}.{dev_pcie_info.bdf_funcid:x}" + ) + + +def _get_device_chip_info(dev_card_id, dev_device_id): + """ + Get the device's chip information. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The chip information, carrying at least chip_name and aicore_cnt. + + """ + try: + return pydcmi.dcmi_get_device_chip_info_v2(dev_card_id, dev_device_id) + except pydcmi.DCMIError: + # The binding's V2 wrapper already falls back when the symbol is + # missing; the operator's ChipInfoHandler falls back on any failure, + # which an older driver rejecting the V2 struct produces. + return pydcmi.dcmi_get_device_chip_info(dev_card_id, dev_device_id) + + def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]: """ Get device memory information. @@ -357,35 +541,57 @@ def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]: try: dev_hbm_info = pydcmi.dcmi_get_device_hbm_info(dev_card_id, dev_device_id) if dev_hbm_info.memory_size > 0: - dev_mem = dev_hbm_info.memory_size - dev_mem_used = dev_hbm_info.memory_usage - else: - dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( - dev_card_id, - dev_device_id, - ) - dev_mem = dev_memory_info.memory_size - dev_mem_used = ( - dev_memory_info.memory_size - dev_memory_info.memory_available - ) + return dev_hbm_info.memory_size, dev_hbm_info.memory_usage except pydcmi.DCMIError as e: - if e.value in [ + if e.value not in [ pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND, pydcmi.DCMI_ERROR_NOT_SUPPORT, pydcmi.DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER, ]: - dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( - dev_card_id, - dev_device_id, - ) - dev_mem = dev_memory_info.memory_size - dev_mem_used = ( - dev_memory_info.memory_size - dev_memory_info.memory_available - ) - else: raise - return dev_mem, dev_mem_used + return _get_device_memory_info_without_hbm(dev_card_id, dev_device_id) + + +def _get_device_memory_info_without_hbm(dev_card_id, dev_device_id) -> tuple[int, int]: + """ + Get device memory information from the non-HBM calls. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + A tuple containing total memory and used memory in MiB. + + """ + try: + dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( + dev_card_id, + dev_device_id, + ) + except pydcmi.DCMIError: + # An older driver exposes the V2 call only, as the operator's + # MemoryHandler.V2 uses. + dev_memory_info_v2 = pydcmi.dcmi_get_device_memory_info_v2( + dev_card_id, + dev_device_id, + ) + dev_mem = dev_memory_info_v2.memory_size + # Divergence from the operator, deliberate: it computes + # `memory_size - memory_available` here too, but the V2 struct has no + # available figure at all and its conversion leaves that field zero, + # so it reports every card as fully used -- indistinguishable from a + # real out-of-memory condition. The utilization percentage the struct + # does carry is the only used-memory signal on this path. + return dev_mem, dev_mem * dev_memory_info_v2.utiliza // 100 + + return ( + dev_memory_info.memory_size, + dev_memory_info.memory_size - dev_memory_info.memory_available, + ) def _get_device_memory_status(dev_card_id, dev_device_id) -> DeviceMemoryStatusEnum: @@ -450,34 +656,6 @@ def _get_device_roce_network_info( return ip, mask, gateway -def _get_device_virtual_info( - dev_card_id, - dev_device_id, -) -> pydcmi.c_dcmi_vdev_query_stru | None: - """ - Get device virtual information. - - Returns: - A c_dcmi_vdev_query_stru object if successful, None otherwise. - - """ - try: - c_vdev_query_stru = pydcmi.c_dcmi_vdev_query_stru() - pydcmi.dcmi_get_device_info( - dev_card_id, - dev_device_id, - pydcmi.DCMI_MAIN_CMD_VDEV_MNG, - pydcmi.DCMI_VMNG_SUB_CMD_GET_VDEV_RESOURCE, - c_vdev_query_stru, - ) - except pydcmi.DCMIError: - debug_log_exception(logger, "Failed to get device virtual info") - else: - return c_vdev_query_stru - - return None - - def _get_toolkit_home() -> Path: """ Resolve the Ascend toolkit home directory. diff --git a/gpustack_runtime/detector/pydcmi/__init__.py b/gpustack_runtime/detector/pydcmi/__init__.py index 9e54eb1..7c35e68 100644 --- a/gpustack_runtime/detector/pydcmi/__init__.py +++ b/gpustack_runtime/detector/pydcmi/__init__.py @@ -389,6 +389,21 @@ class c_dcmi_chip_info_v2(_PrintableStructure): ] +class c_dcmi_pcie_info(_PrintableStructure): + # The V1 struct, which -- unlike c_dcmi_pcie_info_all -- carries no domain, + # and orders the ids differently. Both are as dcmi_interface_api.h declares + # them. + _fields_: ClassVar = [ + ("deviceid", c_uint), + ("venderid", c_uint), + ("subvenderid", c_uint), + ("subdeviceid", c_uint), + ("bdf_deviceid", c_uint), + ("bdf_busid", c_uint), + ("bdf_funcid", c_uint), + ] + + class c_dcmi_pcie_info_all(_PrintableStructure): _fields_: ClassVar = [ ("venderid", c_uint), @@ -433,6 +448,16 @@ class c_dcmi_hbm_info(_PrintableStructure): ] +class c_dcmi_memory_info(_PrintableStructure): + # The V2 struct, which carries no available memory, only the utilization + # percentage, unlike c_dcmi_get_memory_info_stru of V3. + _fields_: ClassVar = [ + ("memory_size", c_ulonglong), + ("freq", c_uint), + ("utiliza", c_uint), + ] + + class c_dcmi_get_memory_info_stru(_PrintableStructure): _fields_: ClassVar = [ ("memory_size", c_ulonglong), @@ -847,6 +872,14 @@ def dcmi_get_device_type(card_id, device_id): return c_device_type.value +def dcmi_get_device_pcie_info(card_id, device_id): + c_pcie_info = c_dcmi_pcie_info() + fn = _dcmiGetFunctionPointer("dcmi_get_device_pcie_info") + ret = fn(card_id, device_id, byref(c_pcie_info)) + _dcmiCheckReturn(ret) + return c_pcie_info + + def dcmi_get_device_pcie_info_v2(card_id, device_id): c_pcie_info = c_dcmi_pcie_info_all() fn = _dcmiGetFunctionPointer("dcmi_get_device_pcie_info_v2") @@ -951,6 +984,14 @@ def dcmi_get_device_hbm_info(card_id, device_id): return c_hbm_info +def dcmi_get_device_memory_info_v2(card_id, device_id): + c_memory_info = c_dcmi_memory_info() + fn = _dcmiGetFunctionPointer("dcmi_get_device_memory_info_v2") + ret = fn(card_id, device_id, byref(c_memory_info)) + _dcmiCheckReturn(ret) + return c_memory_info + + def dcmi_get_device_memory_info_v3(card_id, device_id): c_memory_info = c_dcmi_get_memory_info_stru() fn = _dcmiGetFunctionPointer("dcmi_get_device_memory_info_v3") @@ -1119,6 +1160,17 @@ def dcmi_get_npu_work_mode(card_id): return c_work_mode.value +def dcmi_get_device_die(card_id, device_id): + # The V1 call takes no die type: it reports the SoC die only, which is what + # dcmi_get_device_die_v2 returns for DCMI_DIE_TYPE_VDIE. The struct is + # dcmi_soc_die_stru, laid out exactly as dcmi_die_id. + c_die_id = c_dcmi_die_id() + fn = _dcmiGetFunctionPointer("dcmi_get_device_die") + ret = fn(card_id, device_id, byref(c_die_id)) + _dcmiCheckReturn(ret) + return " ".join([hex(i)[2:] for i in c_die_id.soc_die]) + + def dcmi_get_device_die_v2(card_id, device_id, input_type): c_die_id = c_dcmi_die_id() fn = _dcmiGetFunctionPointer("dcmi_get_device_die_v2") diff --git a/tests/gpustack_runtime/detector/test_ascend.py b/tests/gpustack_runtime/detector/test_ascend.py index 461210b..1851f7a 100644 --- a/tests/gpustack_runtime/detector/test_ascend.py +++ b/tests/gpustack_runtime/detector/test_ascend.py @@ -1,5 +1,21 @@ +from __future__ import annotations + +import ctypes +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import ascend as cdi_ascend +from gpustack_runtime.deployer.cdi.ascend import AscendGenerator +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + ascend, + pydcmi, +) +from gpustack_runtime.detector.__utils__ import get_utilization from gpustack_runtime.detector.ascend import AscendDetector @@ -21,3 +37,699 @@ def test_get_topology(): det = AscendDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# ABI of the V1 structs, pinned against dcmi_interface_api.h. # +# # +# No Ascend driver exists in CI, so nothing else would catch a mislaid field: # +# the numbers below are read off the header's declarations, not off the # +# ctypes code they check. # +# --------------------------------------------------------------------------- # + + +def test_pcie_info_v1_struct_matches_the_header(): + # struct dcmi_pcie_info { unsigned int deviceid, venderid, subvenderid, + # subdeviceid, bdf_deviceid, bdf_busid, bdf_funcid; } + assert [f[0] for f in pydcmi.c_dcmi_pcie_info._fields_] == [ + "deviceid", + "venderid", + "subvenderid", + "subdeviceid", + "bdf_deviceid", + "bdf_busid", + "bdf_funcid", + ] + assert ctypes.sizeof(pydcmi.c_dcmi_pcie_info) == 7 * 4 + for offset, name in enumerate( + [f[0] for f in pydcmi.c_dcmi_pcie_info._fields_], + ): + assert getattr(pydcmi.c_dcmi_pcie_info, name).offset == offset * 4 + # The V1 struct carries no PCI domain, where the V2 one does. The asymmetry + # is why the detector formats a domain of 0 on the fallback path. + assert not hasattr(pydcmi.c_dcmi_pcie_info, "domain") + assert hasattr(pydcmi.c_dcmi_pcie_info_all, "domain") + + +def test_memory_info_v2_struct_matches_the_header(): + # struct dcmi_memory_info { unsigned long long memory_size; unsigned int + # freq; unsigned int utiliza; } + assert [f[0] for f in pydcmi.c_dcmi_memory_info._fields_] == [ + "memory_size", + "freq", + "utiliza", + ] + assert pydcmi.c_dcmi_memory_info.memory_size.offset == 0 + assert pydcmi.c_dcmi_memory_info.memory_size.size == 8 + assert pydcmi.c_dcmi_memory_info.freq.offset == 8 + assert pydcmi.c_dcmi_memory_info.utiliza.offset == 12 + assert ctypes.sizeof(pydcmi.c_dcmi_memory_info) == 16 + # Unlike V3, it reports no available memory at all, which is why the used + # memory has to be derived from the utilization percentage. + assert not hasattr(pydcmi.c_dcmi_memory_info, "memory_available") + + +def test_die_id_struct_matches_the_header(): + # The V1 call fills a struct dcmi_soc_die_stru { unsigned int soc_die[5]; }, + # laid out exactly as the dcmi_die_id the V2 call fills. + assert ctypes.sizeof(pydcmi.c_dcmi_die_id) == 5 * 4 + assert pydcmi.c_dcmi_die_id.soc_die.offset == 0 + + +# --------------------------------------------------------------------------- # +# A fake pydcmi carrying a call log, so the type filter, the V1 fallbacks and # +# the information/usage split are provable on a host with no DCMI driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Unit: + """ + One device inside a dcmi card, as the driver would report it. + + The per-generation values are deliberately distinct, so a test can tell + which call a returned value came from. + """ + + card_id: int = 0 + device_id: int = 0 + unit_type: int = pydcmi.DCMI_UNIT_TYPE_NPU + unit_type_readable: bool = True + logic_id: int = 0 + physical_id: int = 0 + physical_id_readable: bool = True + aicore_cnt: int = 20 + ecc_errors: int = 0 + cores_utilization: int = 42 + temperature: int = 55 + power_deciwatts: int = 1234 + # Which generation of calls the driver exposes. + v2_die: bool = True + v2_pcie: bool = True + v2_chip_info: bool = True + hbm: bool = True + v3_memory: bool = True + + @property + def v2_die_id(self) -> str: + return f"1a 2b 3c 4d {self.logic_id:x}" + + @property + def v1_die_id(self) -> str: + return f"9f 8e 7d 6c {self.logic_id:x}" + + @property + def v2_chip_name(self) -> str: + return "910B3" + + @property + def v1_chip_name(self) -> str: + return "910A" + + @property + def v2_bdf(self) -> str: + return f"0001:{0x10 + self.logic_id:02x}:00.0" + + @property + def v1_bdf(self) -> str: + # The V1 struct has no domain, so the detector formats it as zero. + return f"0000:{0x20 + self.logic_id:02x}:00.0" + + +class _FakeStruct: + """ + A stand-in for a ctypes struct the binding would return. + """ + + def __init__(self, **fields): + self.__dict__.update(fields) + + +@dataclass +class _FakeDCMI: + """ + A stand-in for the pydcmi binding, recording every call the detector makes. + + The error type and the enumeration constants are the real module's, so a + fake drifting from the binding's contract fails here rather than on + hardware. Entry points are dispatched by name, as test_metax.py's fake + does, which keeps the driver's own naming out of the handlers' names. + + The vdev (vNPU) constants are deliberately *not* exposed: a reintroduced + vNPU branch fails here with an AttributeError rather than passing silently. + """ + + units: list[_Unit] = field(default_factory=lambda: [_Unit()]) + calls: list[str] = field(default_factory=list) + + DCMIError = pydcmi.DCMIError + DCMI_UNIT_TYPE_NPU = pydcmi.DCMI_UNIT_TYPE_NPU + DCMI_UNIT_TYPE_MCU = pydcmi.DCMI_UNIT_TYPE_MCU + DCMI_DIE_TYPE_VDIE = pydcmi.DCMI_DIE_TYPE_VDIE + DCMI_DEVICE_TYPE_HBM = pydcmi.DCMI_DEVICE_TYPE_HBM + DCMI_DEVICE_TYPE_DDR = pydcmi.DCMI_DEVICE_TYPE_DDR + DCMI_INPUT_TYPE_AICORE = pydcmi.DCMI_INPUT_TYPE_AICORE + DCMI_PORT_TYPE_ROCE_PORT = pydcmi.DCMI_PORT_TYPE_ROCE_PORT + DCMI_ERROR_FUNCTION_NOT_FOUND = pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND + DCMI_ERROR_NOT_SUPPORT = pydcmi.DCMI_ERROR_NOT_SUPPORT + DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER = pydcmi.DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER + + # Memory, in MiB, per generation of the memory calls. + hbm_size: int = 65536 + hbm_usage: int = 1024 + v3_size: int = 32768 + v3_available: int = 24576 + v2_size: int = 16384 + v2_utiliza: int = 25 + + def __getattr__(self, name: str): + handler = { + "dcmi_init": self._init, + "dcmi_get_driver_version": self._get_driver_version, + "dcmi_get_card_list": self._get_card_list, + "dcmi_get_device_num_in_card": self._get_device_num_in_card, + "dcmi_get_device_type": self._get_device_type, + "dcmi_get_device_die_v2": self._get_device_die_v2, + "dcmi_get_device_die": self._get_device_die, + "dcmi_get_device_chip_info_v2": self._get_device_chip_info_v2, + "dcmi_get_device_chip_info": self._get_device_chip_info, + "dcmi_get_device_hbm_info": self._get_device_hbm_info, + "dcmi_get_device_memory_info_v3": self._get_device_memory_info_v3, + "dcmi_get_device_memory_info_v2": self._get_device_memory_info_v2, + "dcmi_get_device_ecc_info": self._get_device_ecc_info, + "dcmi_get_device_logic_id": self._get_device_logic_id, + "dcmi_get_device_phyid_from_logicid": self._get_phyid_from_logicid, + "dcmi_get_device_bdf": self._get_device_bdf, + "dcmi_get_device_pcie_info": self._get_device_pcie_info, + "dcmi_get_device_ip": self._get_device_ip, + "dcmi_get_affinity_cpu_info_by_device_id": self._get_affinity_cpu_info, + "dcmi_get_device_utilization_rate": self._get_device_utilization_rate, + "dcmi_get_device_temperature": self._get_device_temperature, + "dcmi_get_device_power_info": self._get_device_power_info, + }.get(name) + if handler is None: + msg = f"module pydcmi has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _unit(self, card_id: int, device_id: int) -> _Unit: + for unit in self.units: + if unit.card_id == card_id and unit.device_id == device_id: + return unit + msg = f"no such device: card {card_id}, device {device_id}" + raise AssertionError(msg) + + def _unsupported(self) -> DCMIError: + return self.DCMIError(self.DCMI_ERROR_NOT_SUPPORT) + + def _init(self) -> None: + pass + + def _get_driver_version(self) -> str: + return "24.1.0" + + def _get_card_list(self) -> tuple[int, list[int]]: + cards = sorted({unit.card_id for unit in self.units}) + return len(cards), cards + + def _get_device_num_in_card(self, card_id: int) -> int: + return len([unit for unit in self.units if unit.card_id == card_id]) + + def _get_device_type(self, card_id: int, device_id: int) -> int: + unit = self._unit(card_id, device_id) + if not unit.unit_type_readable: + raise self._unsupported() + return unit.unit_type + + def _get_device_die_v2(self, card_id: int, device_id: int, input_type: int) -> str: + assert input_type == pydcmi.DCMI_DIE_TYPE_VDIE + unit = self._unit(card_id, device_id) + if not unit.v2_die: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.v2_die_id + + def _get_device_die(self, card_id: int, device_id: int) -> str: + return self._unit(card_id, device_id).v1_die_id + + def _get_device_chip_info_v2(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.v2_chip_info: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return _FakeStruct(chip_name=unit.v2_chip_name, aicore_cnt=unit.aicore_cnt) + + def _get_device_chip_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct(chip_name=unit.v1_chip_name, aicore_cnt=unit.aicore_cnt) + + def _get_device_hbm_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.hbm: + raise self._unsupported() + return _FakeStruct(memory_size=self.hbm_size, memory_usage=self.hbm_usage) + + def _get_device_memory_info_v3(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.v3_memory: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return _FakeStruct( + memory_size=self.v3_size, + memory_available=self.v3_available, + ) + + def _get_device_memory_info_v2(self, card_id: int, device_id: int) -> _FakeStruct: + return _FakeStruct(memory_size=self.v2_size, utiliza=self.v2_utiliza) + + def _get_device_ecc_info( + self, + card_id: int, + device_id: int, + device_type: int, + ) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct( + enable_flag=1, + single_bit_error_cnt=unit.ecc_errors, + double_bit_error_cnt=0, + ) + + def _get_device_logic_id(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).logic_id + + def _get_phyid_from_logicid(self, logic_id: int) -> int: + for unit in self.units: + if unit.logic_id == logic_id: + if not unit.physical_id_readable: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.physical_id + msg = f"no such logic id: {logic_id}" + raise AssertionError(msg) + + def _get_device_bdf(self, card_id: int, device_id: int) -> str: + unit = self._unit(card_id, device_id) + if not unit.v2_pcie: + # As the real wrapper does: it is built on the V2 call. + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.v2_bdf + + def _get_device_pcie_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct( + bdf_busid=0x20 + unit.logic_id, + bdf_deviceid=0, + bdf_funcid=0, + ) + + def _get_device_ip(self, card_id: int, device_id: int, port_type: int) -> None: + raise self._unsupported() + + def _get_affinity_cpu_info(self, card_id: int, device_id: int) -> None: + raise self._unsupported() + + def _get_device_utilization_rate( + self, + card_id: int, + device_id: int, + input_type: int, + ) -> int: + assert input_type == pydcmi.DCMI_INPUT_TYPE_AICORE + return self._unit(card_id, device_id).cores_utilization + + def _get_device_temperature(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).temperature + + def _get_device_power_info(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).power_deciwatts + + +_USAGE_ONLY_CALLS = { + "dcmi_get_device_utilization_rate", + "dcmi_get_device_temperature", + "dcmi_get_device_power_info", +} + + +@pytest.fixture(autouse=True) +def _reset_is_supported_cache(): + # is_supported()/detect_pci_devices() are lru_cache'd, so a value + # observed by one test would otherwise leak into the next. + AscendDetector.is_supported.cache_clear() + AscendDetector.detect_pci_devices.cache_clear() + yield + AscendDetector.is_supported.cache_clear() + AscendDetector.detect_pci_devices.cache_clear() + + +@pytest.fixture +def fake_pydcmi(monkeypatch): + def _install(units: list[_Unit] | None = None, **kwargs) -> _FakeDCMI: + fake = _FakeDCMI(units=units if units is not None else [_Unit()], **kwargs) + monkeypatch.setattr(ascend, "pydcmi", fake) + # No PCI sysfs tree exists on the dev machine, so bypass the PCI + # presence check that is_supported() otherwise gates on. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: the NPU type filter. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_skips_a_non_npu_device(fake_pydcmi): + fake = fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=0, + device_id=1, + logic_id=1, + physical_id=1, + unit_type=pydcmi.DCMI_UNIT_TYPE_MCU, + ), + ], + ) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + # The skipped unit is never queried any further. + assert fake.calls.count("dcmi_get_device_die_v2") == 1 + + +def test_detect_info_keeps_a_device_whose_type_is_unreadable(fake_pydcmi): + # The operator skips a device only when the type call *succeeds* and + # reports something other than an NPU. + fake_pydcmi([_Unit(unit_type_readable=False)]) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + + +# --------------------------------------------------------------------------- # +# detect_info: the V1 fallbacks, one test each, on value and call log. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_v2_calls(fake_pydcmi): + fake = fake_pydcmi() + unit = fake.units[0] + + devices = AscendDetector().detect_info() + + assert devices[0].uuid == unit.v2_die_id.upper() + assert devices[0].name == unit.v2_chip_name + assert devices[0].appendix["bdf"] == unit.v2_bdf + assert devices[0].memory == fake.hbm_size + for v1_call in ( + "dcmi_get_device_die", + "dcmi_get_device_chip_info", + "dcmi_get_device_pcie_info", + "dcmi_get_device_memory_info_v3", + "dcmi_get_device_memory_info_v2", + ): + assert v1_call not in fake.calls + + +def test_detect_info_falls_back_to_the_v1_die(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_die=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].uuid == fake.units[0].v1_die_id.upper() + assert "dcmi_get_device_die_v2" in fake.calls + assert "dcmi_get_device_die" in fake.calls + + +def test_detect_info_falls_back_to_the_v1_pcie_info(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_pcie=False)]) + + devices = AscendDetector().detect_info() + + # The V1 struct carries no domain, so it reads as 0000. + assert devices[0].appendix["bdf"] == fake.units[0].v1_bdf + assert "dcmi_get_device_bdf" in fake.calls + assert "dcmi_get_device_pcie_info" in fake.calls + + +def test_detect_info_falls_back_to_the_v1_chip_info(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_chip_info=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].name == fake.units[0].v1_chip_name + assert devices[0].cores == fake.units[0].aicore_cnt + assert "dcmi_get_device_chip_info_v2" in fake.calls + assert "dcmi_get_device_chip_info" in fake.calls + + +def test_detect_info_falls_back_to_the_v3_memory_when_hbm_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].memory == fake.v3_size + assert "dcmi_get_device_memory_info_v3" in fake.calls + assert "dcmi_get_device_memory_info_v2" not in fake.calls + + +def test_detect_info_falls_back_to_the_v2_memory_when_v3_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False, v3_memory=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].memory == fake.v2_size + assert "dcmi_get_device_memory_info_v2" in fake.calls + + +def test_detect_info_yields_a_device_on_a_v1_only_driver(fake_pydcmi): + # The whole point of the fallbacks: a driver exposing none of the newer + # calls still detects. + fake = fake_pydcmi( + [ + _Unit( + v2_die=False, + v2_pcie=False, + v2_chip_info=False, + hbm=False, + v3_memory=False, + ), + ], + ) + unit = fake.units[0] + + devices = AscendDetector().detect_info() + + assert len(devices) == 1 + assert devices[0].uuid == unit.v1_die_id.upper() + assert devices[0].name == unit.v1_chip_name + assert devices[0].appendix["bdf"] == unit.v1_bdf + assert devices[0].memory == fake.v2_size + + +# --------------------------------------------------------------------------- # +# detect_info: no vGPU, no usage calls. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_has_no_vgpu_in_appendix(fake_pydcmi): + fake_pydcmi() + + devices = AscendDetector().detect_info() + + assert all("vgpu" not in dev.appendix for dev in devices) + + +def test_detect_info_keeps_the_physical_id_appendix(fake_pydcmi): + fake_pydcmi([_Unit(logic_id=1, physical_id=6)]) + + devices = AscendDetector().detect_info() + + assert devices[0].index == 1 + assert devices[0].appendix["physical_id"] == 6 + + +def test_detect_info_skips_a_device_without_a_readable_physical_id(fake_pydcmi): + # /dev/davinciN is numbered by the physical id, so a device whose physical + # id the driver will not answer cannot be addressed. Reporting it would + # offer an NPU that no CDI spec can reference, and standing the logic id in + # for the physical id would reference another NPU, so it is dropped -- as + # the operator's ascend/device.go does on a failed GetPhysicalID. + fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=1, + device_id=0, + logic_id=1, + physical_id=7, + physical_id_readable=False, + ), + ], + ) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + + +def test_detect_info_issues_no_usage_calls(fake_pydcmi): + fake = fake_pydcmi() + + AscendDetector().detect_info() + + assert _USAGE_ONLY_CALLS.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# detect_usage: merged by uuid, memory_status recomputed. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_the_usage_fields_by_uuid(fake_pydcmi): + fake = fake_pydcmi() + unit = fake.units[0] + + devices = AscendDetector().detect_info() + result = AscendDetector().detect_usage(devices) + + assert result is devices + assert devices[0].cores_utilization == unit.cores_utilization + assert devices[0].memory_used == fake.hbm_usage + assert devices[0].memory_utilization == get_utilization( + fake.hbm_usage, + fake.hbm_size, + ) + assert devices[0].temperature == unit.temperature + assert devices[0].power_used == unit.power_deciwatts / 10 + + +def test_detect_usage_recomputes_the_memory_status(fake_pydcmi, monkeypatch): + # merge_devices_usage overwrites memory_status, so the usage pass has to + # produce it as well or the inventory pass's verdict is lost. + fake_pydcmi([_Unit(ecc_errors=3)]) + # The ECC read costs a driver call per device, so the health check is off + # by default. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + + devices = AscendDetector().detect_info() + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + AscendDetector().detect_usage(devices) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_usage_keeps_a_healthy_status_healthy(fake_pydcmi): + fake_pydcmi() + + devices = AscendDetector().detect_usage(AscendDetector().detect_info()) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_usage_skips_a_non_npu_device(fake_pydcmi): + fake = fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=0, + device_id=1, + logic_id=1, + physical_id=1, + unit_type=pydcmi.DCMI_UNIT_TYPE_MCU, + ), + ], + ) + + devices = AscendDetector().detect_info() + fake.calls.clear() + AscendDetector().detect_usage(devices) + + # The filter applies to the usage pass too, as it does in the operator's + # MonitorAccelerator. + assert fake.calls.count("dcmi_get_device_utilization_rate") == 1 + + +def test_detect_usage_falls_back_to_the_v2_memory_when_v3_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False, v3_memory=False)]) + + devices = AscendDetector().detect_usage(AscendDetector().detect_info()) + + # The V2 struct reports no available memory, so the used memory comes from + # its utilization percentage -- the operator instead subtracts an + # always-zero available figure and so reports the card as fully used. + assert devices[0].memory_used == fake.v2_size * fake.v2_utiliza // 100 + assert devices[0].memory_used != fake.v2_size + + +def test_detect_composes_info_and_usage_by_default(fake_pydcmi): + fake = fake_pydcmi() + + devices = AscendDetector().detect() + + assert devices[0].cores_utilization == fake.units[0].cores_utilization + assert devices[0].power_used == fake.units[0].power_deciwatts / 10 + assert "vgpu" not in devices[0].appendix + + +def test_detect_without_usage_issues_no_usage_calls(fake_pydcmi): + fake = fake_pydcmi() + + devices = AscendDetector().detect(usage=False) + + assert len(devices) == 1 + assert _USAGE_ONLY_CALLS.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# CDI: /dev/davinci{N} only, from the appendix physical id. # +# --------------------------------------------------------------------------- # + + +def test_cdi_emits_no_vdavinci_path(monkeypatch): + seen_paths: list[str] = [] + + def _fake_device_node(path, **_kwargs): + seen_paths.append(path) + return {"path": path} + + monkeypatch.setattr(cdi_ascend, "device_to_cdi_device_node", _fake_device_node) + monkeypatch.setattr(cdi_ascend, "path_to_cdi_mount", lambda **_kwargs: None) + + devices = [ + # A stale vgpu appendix must not resurrect the /dev/vdavinci path. + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=0, + name="910B3", + uuid="DIE-0", + memory=65536, + appendix={"card_id": 0, "device_id": 0, "physical_id": 3, "vgpu": True}, + ), + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=1, + name="910B3", + uuid="DIE-1", + memory=65536, + appendix={"card_id": 1, "device_id": 0}, + ), + ] + + config = AscendGenerator().generate(devices) + + assert config is not None + assert not any("vdavinci" in path for path in seen_paths) + # The device carrying a physical id is addressed by it. + assert "/dev/davinci3" in seen_paths + # The device without one is skipped: Device.index is the logic id, so + # standing it in for the physical id would address another NPU's node. + assert "/dev/davinci1" not in seen_paths From b381c42d2a1d5c62f6249d4f971c123eec2cbab3 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:08 +0800 Subject: [PATCH 04/12] feat(detector): detect cambricon mlus through cndev instead of parsing cnmon - add `pycndev`, a hand-written ctypes binding over libcndev following the pydcmi pattern: 15 structs and the calls a Cambricon detector needs, plus the operator's NEUWARE_HOME search order and its two synthetic error codes - pin every struct's size and field offsets, the API version, the error codes and the device types against the driver's own header, since no hardware in CI can catch a layout or constant mistake later, and stamp the temperature structure's IN version field, which the header requires and the operator's binding omits - keep the module inert at import: a missing library raises the binding's own error with the loader's OSError chained onto it, never a bare OSError, so a present but unloadable library stops reading as an absent one - hold the load lock across the whole check-then-initialize sequence, as the operator's binding does with sync.Once: checking `_libInitialized` outside it let two concurrent first callers both reach cndevInit, and the loser's already-initialized error was cached as the library's permanent state. The lock is reentrant because `_cndevGetFunctionPointer` takes it too, and the init flags are cleared only after the driver has let go of what cndevInit allocated. - report no identity rather than manufacturing a bare "MLU-" from an empty field, which is exactly the ambiguous uuid the usage join has to drop - derive overallHealth in the V1 health fallback instead of leaving it at zero, which is CNDEV_HEALTH_RESULT_PASS - replace the `cnmon info -e -m -u -j` shell-out with the binding, so a host with a working driver no longer needs the command-line tool present, and report the driver and Neuware versions, the PCIe bus id, the NUMA node and a real health state, none of which the shell-out surfaced - move core utilization, used memory, temperature and power into `detect_usage`, and drop the `vgpu` appendix key, the last one in the package - skip a card whose required reads fail, as the operator does, without renumbering the cards that answered -- but fail the pass when every card is skipped, which is systemic rather than one faulty card, and used to report a host full of MLUs as having none - keep the binding out of the linter's naming and star-import rules, as every other hand-written binding here is Signed-off-by: thxCode --- gpustack_runtime/detector/cambricon.py | 303 +++++- gpustack_runtime/detector/pycndev/__init__.py | 840 ++++++++++++++++ ruff.toml | 1 + .../detector/test_cambricon.py | 545 +++++++++++ .../gpustack_runtime/detector/test_pycndev.py | 926 ++++++++++++++++++ 5 files changed, 2571 insertions(+), 44 deletions(-) create mode 100644 gpustack_runtime/detector/pycndev/__init__.py create mode 100644 tests/gpustack_runtime/detector/test_pycndev.py diff --git a/gpustack_runtime/detector/cambricon.py b/gpustack_runtime/detector/cambricon.py index 958077f..b1a5f0a 100644 --- a/gpustack_runtime/detector/cambricon.py +++ b/gpustack_runtime/detector/cambricon.py @@ -1,29 +1,48 @@ from __future__ import annotations as __future_annotations__ -import json +import contextlib import logging +import re from functools import lru_cache +from pathlib import Path from .. import envs -from ..logging import debug_log_exception -from . import DeviceMemoryStatusEnum -from .__types__ import Detector, Device, Devices, ManufacturerEnum +from ..logging import debug_log_exception, debug_log_warning +from . import pycndev +from .__types__ import ( + Detector, + Device, + DeviceMemoryStatusEnum, + Devices, + ManufacturerEnum, + merge_devices_usage, +) from .__utils__ import ( PCIDevice, - execute_shell_command, + get_brief_version, + get_numa_node_by_bdf, get_pci_devices, get_utilization, - safe_float, - safe_int, - support_command, ) logger = logging.getLogger(__name__) +_NEUWARE_VERSION_PATH = Path("/usr/local/neuware/version.txt") +""" +Where the CNToolkit package records the Neuware version, i.e. the runtime +version. The single path the operator's getRuntimeVersion stats. +""" + +_NEUWARE_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") +""" +The version pattern the operator matches within that file, so a decorated line +still yields the version alone. +""" + class CambriconDetector(Detector): """ - Detect Cambricon GPUs. + Detect Cambricon MLUs. """ @staticmethod @@ -46,7 +65,11 @@ def is_supported() -> bool: logger.debug("No Cambricon PCI devices found") return supported - supported = support_command("cnmon") + try: + pycndev.cndevInit() + supported = True + except Exception: + debug_log_exception(logger, "Failed to initialize CNDev") return supported @@ -62,12 +85,12 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.CAMBRICON) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Cambricon GPUs using cnmon tool. + Detect Cambricon MLUs' inventory using pycndev, without usage metrics. Returns: - A list of detected Cambricon GPU devices, + A list of detected Cambricon MLU devices, or None if not supported. Raises: @@ -80,54 +103,246 @@ def detect(self) -> Devices | None: ret: Devices = [] try: - output = execute_shell_command( - "cnmon info -e -m -u -j > /dev/null && cat cnmon_info.json", - ) - """ - Example output: - TODO(thxCode): Add example output here. - """ - - output_json = json.loads(output) - dev_infos = output_json.get("CnmonInfo", []) - for dev_info in dev_infos: - dev_index = safe_int(dev_info.get("CardNum")) - dev_name = dev_info.get("ProductName") - dev_uuid = dev_info.get("UUID") - - dev_util_info = dev_info.get("Utilization", {}) - dev_cores_util = safe_float(dev_util_info.get("MLUAverage", 0)) - - dev_mem_usage_info = dev_info.get("PhysicalMemUsage", {}) - dev_mem = safe_int(dev_mem_usage_info.get("Total", 0)) - dev_mem_used = safe_int(dev_mem_usage_info.get("Used", 0)) - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - - dev_temp_info = dev_info.get("Temperature", {}) - dev_temp = safe_float(dev_temp_info.get("Chip", 0)) + pycndev.cndevInit() + + # The Neuware version is a host-wide file read, not a per-device + # driver call, so it is read once above the loop, as the operator's + # DetectAccelerator does. + sys_runtime_ver_original = _get_runtime_version() + sys_runtime_ver = get_brief_version(sys_runtime_ver_original) + + dev_count = pycndev.cndevGetDeviceCount() + dev_skipped_error = None + dev_skipped = 0 + for dev_idx in range(dev_count): + try: + dev = pycndev.cndevGetDeviceHandleByIndex(dev_idx) + dev_uuid = pycndev.cndevGetUUID(dev) + dev_name = pycndev.cndevGetCardNameByDevId(dev) + dev_mem_info = pycndev.cndevGetMemoryUsageV2(dev) + dev_bdf = pycndev.cndevGetPCIeBusId(dev) + except pycndev.CNDevError as e: + # A card whose identity or inventory cannot be read is + # skipped, as the operator's DetectAccelerator does: one + # faulty card must cost that card, not every card of the + # host. Skipping does not renumber the survivors -- an index + # is what the driver enumerated the card at. + debug_log_warning( + logger, + "Failed to fetch device %d, skipping it", + dev_idx, + ) + dev_skipped_error = e + dev_skipped += 1 + continue + + # cndev.h calls the unit MB, but the operator assigns it straight + # into its MiB memory field, cnmon reports MiB, and this repo's + # Ascend detector treats its own MB-declared sizes the same way. + # Converting would under-report ~4.8% against the operator on the + # same host, which is the very discrepancy Story 1 removes. + dev_mem = dev_mem_info.physicalMemoryTotal + dev_mem_status = _get_memory_status(dev) + + dev_driver_ver = None + with contextlib.suppress(pycndev.CNDevError): + dev_ver_info = pycndev.cndevGetVersionInfo(dev) + # The operator formats major.minor only; the build number + # comes free from the same query and is the digit a driver + # bug is diagnosed by, so it is kept. + dev_driver_ver = ( + f"{dev_ver_info.driverMajorVersion}" + f".{dev_ver_info.driverMinorVersion}" + f".{dev_ver_info.driverBuildVersion}" + ) + + dev_numa = get_numa_node_by_bdf(dev_bdf) + if not dev_numa: + with contextlib.suppress(pycndev.CNDevError): + dev_numa_node_id = pycndev.cndevGetNUMANodeIdByDevId(dev) + if dev_numa_node_id.nodeId >= 0: + dev_numa = str(dev_numa_node_id.nodeId) dev_appendix = { - "vgpu": False, + "bdf": dev_bdf, } + if dev_numa: + dev_appendix["numa"] = dev_numa ret.append( Device( manufacturer=self.manufacturer, - index=dev_index, + index=dev_idx, name=dev_name, uuid=dev_uuid, - cores_utilization=dev_cores_util, + driver_version=dev_driver_ver, + runtime_version=sys_runtime_ver, + runtime_version_original=sys_runtime_ver_original, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, appendix=dev_appendix, ), ) + except pycndev.CNDevError: + debug_log_exception(logger, "Failed to fetch devices") + raise except Exception: debug_log_exception(logger, "Failed to process devices fetching") raise + if dev_skipped and dev_skipped == dev_count: + # Skipping one card of several is the graceful degradation this + # vendor is allowed; skipping every one of them is systemic -- a + # driver not exporting a call the loop needs, say -- and reporting it + # as an empty inventory is indistinguishable from a host that has no + # MLU at all. Fail as the other eight vendors do. + raise dev_skipped_error + return ret + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of Cambricon MLUs using pycndev, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pycndev.cndevInit() + + dev_count = pycndev.cndevGetDeviceCount() + dev_skipped_error = None + dev_skipped = 0 + for dev_idx in range(dev_count): + # CNDev enumerates by index, so the whole set is read and + # merge_devices_usage keeps what the caller asked for, as the + # operator's MonitorAccelerator does. + try: + dev = pycndev.cndevGetDeviceHandleByIndex(dev_idx) + dev_uuid = pycndev.cndevGetUUID(dev) + dev_mem_info = pycndev.cndevGetMemoryUsageV2(dev) + except pycndev.CNDevError as e: + debug_log_warning( + logger, + "Failed to fetch device %d usage, skipping it", + dev_idx, + ) + dev_skipped_error = e + dev_skipped += 1 + continue + + dev_mem = dev_mem_info.physicalMemoryTotal + dev_mem_used = dev_mem_info.physicalMemoryUsed + dev_mem_status = _get_memory_status(dev) + + dev_cores_util = 0 + with contextlib.suppress(pycndev.CNDevError): + dev_util_info = pycndev.cndevGetDeviceUtilizationInfo(dev) + dev_cores_util = dev_util_info.averageCoreUtilization + + dev_temp = None + with contextlib.suppress(pycndev.CNDevError): + dev_temp_info = pycndev.cndevGetTemperatureInfo(dev) + dev_temp = dev_temp_info.chip + + dev_power_used = None + with contextlib.suppress(pycndev.CNDevError): + dev_power_info = pycndev.cndevGetDevicePowerInfo(dev) + dev_power_used = dev_power_info.usage + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pycndev.CNDevError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + if dev_skipped and dev_skipped == dev_count: + # Every card refusing its metrics is systemic, not one faulty card, + # and merging nothing would leave the caller reading the information + # query's zeroes as an idle host. + raise dev_skipped_error + + return merge_devices_usage(devices, usages) + + +def _get_memory_status(device: int) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + The verdict is the card-wide health bit, exactly as the operator derives it + (`memoryUnhealthy = healthInfo.Health == 0`): CNDev reports no per-memory + health, and the ECC counters cndevGetECCInfo exposes take no part in it. + + Args: + device: + The device handle. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pycndev.CNDevError): + dev_health_state = pycndev.cndevGetCardHealthStateV2(device) + if dev_health_state.health == 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + +def _get_runtime_version() -> str | None: + """ + Get the Neuware version installed on the host, i.e. the runtime version. + + Returns: + The Neuware version, or None if it cannot be read. + + """ + with contextlib.suppress(OSError): + content = _NEUWARE_VERSION_PATH.read_text().strip() + if match := _NEUWARE_VERSION_PATTERN.search(content): + return match.group() + + return None diff --git a/gpustack_runtime/detector/pycndev/__init__.py b/gpustack_runtime/detector/pycndev/__init__.py new file mode 100644 index 0000000..b0363fc --- /dev/null +++ b/gpustack_runtime/detector/pycndev/__init__.py @@ -0,0 +1,840 @@ +## +# Python bindings for the CNDev library +# +# Derived from the Cambricon MLU driver's cndev.h, as vendored by the operator in +# binding/cndev -- the header of record. Only what the Cambricon detector needs is +# bound: device enumeration, identity, inventory and the usage metrics. MLU-Link, +# sMLU/MIM partitioning, per-process accounting and topology are deliberately absent. +## +from __future__ import annotations as __future_annotations__ + +import os +import string +import sys +import threading +from ctypes import * +from functools import wraps +from typing import ClassVar + +## C Type mappings ## +# cndevDevice_t is an __int32_t handle, not an opaque pointer. +cndevDevice_t = c_int32 + +## Constants ## +CNDEV_VERSION_1 = 1 +CNDEV_VERSION_2 = 2 +CNDEV_VERSION_3 = 3 +CNDEV_VERSION_4 = 4 +CNDEV_VERSION_5 = 5 +CNDEV_VERSION_6 = 6 +CNDEV_UUID_SIZE = 37 +CNDEV_ERR_MSG_LENGTH = 512 +CNDEV_HEALTH_SYSTEM_MAX_INCIDENTS = 64 +# The header hardcodes these array lengths in the struct declarations themselves. +CNDEV_TEMPERATURE_CLUSTER_COUNT = 20 +CNDEV_TEMPERATURE_MEMORY_DIE_COUNT = 8 +CNDEV_UTILIZATION_CORE_COUNT = 80 + +## Enums ## +# A C enum here is int-sized: every enumerator in cndev.h is a small non-negative +# value, so the compiler lays them out as a 4-byte signed int. +_cndevNameEnum_t = c_int32 +CNDEV_DEVICE_TYPE_MLU100 = 0 +CNDEV_DEVICE_TYPE_MLU270 = 1 +CNDEV_DEVICE_TYPE_MLU220_M2 = 16 +CNDEV_DEVICE_TYPE_MLU220_EDGE = 17 +CNDEV_DEVICE_TYPE_MLU220_EVB = 18 +CNDEV_DEVICE_TYPE_MLU220_M2i = 19 +CNDEV_DEVICE_TYPE_MLU290 = 20 +CNDEV_DEVICE_TYPE_MLU370 = 23 +CNDEV_DEVICE_TYPE_MLU365 = 24 +CNDEV_DEVICE_TYPE_CE3226 = 25 +CNDEV_DEVICE_TYPE_MLU590 = 26 +CNDEV_DEVICE_TYPE_MLU585 = 27 +CNDEV_DEVICE_TYPE_1V_2201 = 29 +CNDEV_DEVICE_TYPE_MLU580 = 30 +CNDEV_DEVICE_TYPE_MLU570 = 31 +CNDEV_DEVICE_TYPE_1V_2202 = 32 + +## Enums ## +_cndevEnableStatusEnum_t = c_int32 +CNDEV_FEATURE_DISABLED = 0 +CNDEV_FEATURE_ENABLED = 1 + +## Enums ## +_cndevHealthStateEnum_t = c_int32 +CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM = 0 +CNDEV_HEALTH_STATE_DEVICE_GOOD = 1 + +## Enums ## +_cndevDriverHealthStateEnum_t = c_int32 +CNDEV_HEALTH_STATE_DRIVER_EARLY_INITED = 0 +CNDEV_HEALTH_STATE_DRIVER_BRING_UP = 1 +CNDEV_HEALTH_STATE_DRIVER_BOOTING = 2 +CNDEV_HEALTH_STATE_DRIVER_LATEINIT = 3 +CNDEV_HEALTH_STATE_DRIVER_RUNNING = 4 +CNDEV_HEALTH_STATE_DRIVER_BOOT_ERROR = 5 +CNDEV_HEALTH_STATE_DRIVER_RESET = 6 +CNDEV_HEALTH_STATE_DRIVER_RESET_ERROR = 7 +CNDEV_HEALTH_STATE_DRIVER_UNKNOWN = 8 + +## Enums ## +_cndevHealthResult_t = c_int32 +CNDEV_HEALTH_RESULT_PASS = 0 +CNDEV_HEALTH_RESULT_WARN = 10 +CNDEV_HEALTH_RESULT_FAIL = 20 + +## Enums ## +# The incident systems (cndevHealthSystem_t) and diagnosis codes (cndevHealthError_t) +# are only needed as field widths: nothing on the detect path names an incident, so +# their enumerators are left unbound rather than copied and left to rot. +_cndevHealthSystem_t = c_int32 +_cndevHealthError_t = c_int32 + + +## Error Codes ## +CNDEV_SUCCESS = 0 +CNDEV_ERROR_NO_DRIVER = 1 +CNDEV_ERROR_LOW_DRIVER_VERSION = 2 +CNDEV_ERROR_UNSUPPORTED_API_VERSION = 3 +CNDEV_ERROR_UNINITIALIZED = 4 +CNDEV_ERROR_INVALID_ARGUMENT = 5 +CNDEV_ERROR_INVALID_DEVICE_ID = 6 +CNDEV_ERROR_UNKNOWN = 7 +CNDEV_ERROR_MALLOC = 8 +CNDEV_ERROR_INSUFFICIENT_SPACE = 9 +CNDEV_ERROR_NOT_SUPPORTED = 10 +CNDEV_ERROR_INVALID_LINK = 11 +CNDEV_ERROR_NO_DEVICES = 12 +CNDEV_ERROR_NO_PERMISSION = 13 +CNDEV_ERROR_NOT_FOUND = 14 +CNDEV_ERROR_IN_USE = 15 +CNDEV_ERROR_DUPLICATE = 16 +CNDEV_ERROR_TIMEOUT = 17 +CNDEV_ERROR_IN_PROBLEM = 18 +# Not in the header: the operator's binding/cndev/library.go defines the same two codes, +# at the same values, for the failures that happen before any driver call -- the shared +# object is absent, or the symbol is missing from it. +CNDEV_ERROR_FUNCTION_NOT_FOUND = -99998 +CNDEV_ERROR_LIBRARY_NOT_FOUND = -99999 + +## Lib loading ## +cndevLib = None +# Reentrant, unlike the sibling bindings' lock: cndevInit holds it across the whole +# check-then-initialize sequence, and _cndevGetFunctionPointer takes it as well. +libLoadLock = threading.RLock() +_libInitialized = False +_libInitializedException = None + + +## Error Checking ## +class CNDevError(Exception): + _valClassMapping: ClassVar[dict] = {} + + _errcode_to_string: ClassVar[dict] = { + CNDEV_ERROR_NO_DRIVER: "No Driver", + CNDEV_ERROR_LOW_DRIVER_VERSION: "Low Driver Version", + CNDEV_ERROR_UNSUPPORTED_API_VERSION: "Unsupported API Version", + CNDEV_ERROR_UNINITIALIZED: "Library Not Initialized", + CNDEV_ERROR_INVALID_ARGUMENT: "Invalid Argument", + CNDEV_ERROR_INVALID_DEVICE_ID: "Invalid Device ID", + CNDEV_ERROR_UNKNOWN: "Unknown Error", + CNDEV_ERROR_MALLOC: "Memory Allocation Failed", + CNDEV_ERROR_INSUFFICIENT_SPACE: "Insufficient Space", + CNDEV_ERROR_NOT_SUPPORTED: "Not Supported", + CNDEV_ERROR_INVALID_LINK: "Invalid Link", + CNDEV_ERROR_NO_DEVICES: "No Devices", + CNDEV_ERROR_NO_PERMISSION: "No Permission", + CNDEV_ERROR_NOT_FOUND: "Not Found", + CNDEV_ERROR_IN_USE: "In Use", + CNDEV_ERROR_DUPLICATE: "Duplicate", + CNDEV_ERROR_TIMEOUT: "Time Out", + CNDEV_ERROR_IN_PROBLEM: "In Problem", + CNDEV_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", + CNDEV_ERROR_LIBRARY_NOT_FOUND: "Library Not Found", + } + + def __new__(cls, value): + """ + Maps value to a proper subclass of CNDevError. + See _extractCNDevErrorsAsClasses function for more details. + """ + if cls == CNDevError: + cls = CNDevError._valClassMapping.get(value, cls) + obj = Exception.__new__(cls) + obj.value = value + return obj + + def __str__(self): + return CNDevError._errcode_to_string.get( + self.value, + f"Unknown CNDev Error {self.value}", + ) + + def __eq__(self, other): + if isinstance(other, CNDevError): + return self.value == other.value + if isinstance(other, int): + return self.value == other + return False + + +def cndevExceptionClass(cndevErrorCode): + if cndevErrorCode not in CNDevError._valClassMapping: + msg = f"CNDev error code {cndevErrorCode} is not valid" + raise ValueError(msg) + return CNDevError._valClassMapping[cndevErrorCode] + + +def _extractCNDevErrorsAsClasses(): + """ + Generates a hierarchy of classes on top of CNDevError class. + + Each CNDev Error gets a new CNDevError subclass. This way try,except blocks can + filter appropriate exceptions more easily. + + CNDevError is a parent class. Each CNDEV_ERROR_* gets it's own subclass. + e.g. CNDEV_ERROR_INVALID_ARGUMENT will be turned into CNDevError_InvalidArgument. + """ + this_module = sys.modules[__name__] + cndevErrorsNames = [x for x in dir(this_module) if x.startswith("CNDEV_ERROR_")] + for err_name in cndevErrorsNames: + # e.g. Turn CNDEV_ERROR_INVALID_ARGUMENT into CNDevError_InvalidArgument + class_name = "CNDevError_" + string.capwords( + err_name.replace("CNDEV_ERROR_", ""), + "_", + ).replace("_", "") + err_val = getattr(this_module, err_name) + + def gen_new(val): + def new(typ, *args): + obj = CNDevError.__new__(typ, val) + return obj + + return new + + new_error_class = type(class_name, (CNDevError,), {"__new__": gen_new(err_val)}) + new_error_class.__module__ = __name__ + setattr(this_module, class_name, new_error_class) + CNDevError._valClassMapping[err_val] = new_error_class + + +_extractCNDevErrorsAsClasses() + + +def _cndevCheckReturn(ret): + if ret != CNDEV_SUCCESS: + raise CNDevError(ret) + return ret + + +## Function access ## +_cndevGetFunctionPointer_cache = {} + + +def _cndevGetFunctionPointer(name): + global cndevLib + + if name in _cndevGetFunctionPointer_cache: + return _cndevGetFunctionPointer_cache[name] + + libLoadLock.acquire() + try: + if cndevLib is None: + raise CNDevError(CNDEV_ERROR_UNINITIALIZED) + try: + _cndevGetFunctionPointer_cache[name] = getattr(cndevLib, name) + return _cndevGetFunctionPointer_cache[name] + except AttributeError: + raise CNDevError(CNDEV_ERROR_FUNCTION_NOT_FOUND) + finally: + libLoadLock.release() + + +## Structure definitions ## +class _PrintableStructure(Structure): + """ + Abstract class that produces nicer __str__ output than ctypes.Structure. + """ + + _fmt_ = {} + + def __str__(self): + result = [] + for x in self._fields_: + key = x[0] + value = getattr(self, key) + fmt = "%s" + if key in self._fmt_: + fmt = self._fmt_[key] + elif "" in self._fmt_: + fmt = self._fmt_[""] + result.append(("%s: " + fmt) % (key, value)) + return self.__class__.__name__ + "(" + ", ".join(result) + ")" + + def __getattribute__(self, name): + res = super().__getattribute__(name) + if isinstance(res, bytes): + return res.decode() + return res + + def __setattr__(self, name, value): + if isinstance(value, str): + value = value.encode() + super().__setattr__(name, value) + + +class c_cndevCardInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("number", c_uint32), + ] + + +class c_cndevUUID_t(_PrintableStructure): + # uuid is a NUL-terminated __uint8_t array in the header; c_char keeps the same + # layout and reads back as the string the driver wrote. + _fields_: ClassVar = [ + ("version", c_int32), + ("uuid", c_char * CNDEV_UUID_SIZE), + ("ncsUUID64", c_uint64), + ] + + +class c_cndevMemoryInfoV2_t(_PrintableStructure): + # Sizes are MB, and this struct carries no version field. + _fields_: ClassVar = [ + ("physicalMemoryTotal", c_int64), + ("physicalMemoryUsed", c_int64), + ("reservedMemory", c_int64), + ("virtualMemoryTotal", c_int64), + ("virtualMemoryUsed", c_int64), + ("globalMemory", c_uint64), + ("reserved", c_uint64 * 16), + ] + + +class c_cndevVersionInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("mcuMajorVersion", c_uint32), + ("mcuMinorVersion", c_uint32), + ("mcuBuildVersion", c_uint32), + ("driverMajorVersion", c_uint32), + ("driverMinorVersion", c_uint32), + ("driverBuildVersion", c_uint32), + ] + + +class c_cndevECCInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("oneBitError", c_uint64), + ("multipleOneError", c_uint64), + ("multipleError", c_uint64), + ("multipleMultipleError", c_uint64), + ("correctedError", c_uint64), + ("uncorrectedError", c_uint64), + ("totalError", c_uint64), + ("addressForbiddenError", c_uint64), + ] + + +class c_cndevDevicePowerInfo_t(_PrintableStructure): + # Watts, and this struct carries no version field. + _fields_: ClassVar = [ + ("usage", c_int32), + ("cap", c_int32), + ("machine", c_int32), + ("tdp", c_int32), + ("maxPower", c_int32), + ("reserved", c_int32 * 16), + ] + + +class c_cndevTemperatureInfo_t(_PrintableStructure): + # Degrees Celsius. + _fields_: ClassVar = [ + ("version", c_int32), + ("board", c_int32), + ("cluster", c_int32 * CNDEV_TEMPERATURE_CLUSTER_COUNT), + ("memoryDie", c_int32 * CNDEV_TEMPERATURE_MEMORY_DIE_COUNT), + ("chip", c_int32), + ("airInlet", c_int32), + ("airOutlet", c_int32), + ("memory", c_int32), + ("videoInput", c_int32), + ("cpu", c_int32), + ("isp", c_int32), + ] + + +class c_cndevUtilizationInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("averageCoreUtilization", c_int32), + ("coreUtilization", c_int32 * CNDEV_UTILIZATION_CORE_COUNT), + ] + + +class c_cndevCardName_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("id", _cndevNameEnum_t), + ] + + +class c_cndevPCIeInfoV2_t(_PrintableStructure): + # This struct carries no version field. + _fields_: ClassVar = [ + ("subsystemId", c_uint32), + ("deviceId", c_uint32), + ("vendor", c_uint16), + ("subsystemVendor", c_uint16), + ("domain", c_uint32), + ("bus", c_uint32), + ("device", c_uint32), + ("function", c_uint32), + ("moduleId", c_uint16), + ("slotId", c_uint16), + ("reserved", c_uint32 * 8), + ] + + +class c_cndevCardHealthState_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("health", c_int32), + ("deviceState", _cndevHealthStateEnum_t), + ("driverState", _cndevDriverHealthStateEnum_t), + ] + + +class c_cndevDiagErrorDetail_t(_PrintableStructure): + _fields_: ClassVar = [ + ("msg", c_char * CNDEV_ERR_MSG_LENGTH), + ("device_id", c_uint32), + ("bdf", c_uint32), + ("code", _cndevHealthError_t), + ("category", c_uint32), + ("severity", c_uint32), + ] + + +class c_cndevIncidentInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("system", _cndevHealthSystem_t), + ("health", _cndevHealthResult_t), + ("error", c_cndevDiagErrorDetail_t), + ] + + +class c_cndevCardHealthStateV2_t(_PrintableStructure): + # This struct carries no version field, and the driver fills all 64 incident slots + # in place: a short definition here is a buffer overrun, not a missing field. + _fields_: ClassVar = [ + ("health", c_int32), + ("deviceState", _cndevHealthStateEnum_t), + ("driverState", _cndevDriverHealthStateEnum_t), + ("overallHealth", _cndevHealthResult_t), + ("incident_count", c_uint32), + ("incidents", c_cndevIncidentInfo_t * CNDEV_HEALTH_SYSTEM_MAX_INCIDENTS), + ("reserved", c_uint32 * 8), + ] + + +class c_cndevNUMANodeId_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("nodeId", c_int32), + ] + + +## Device name fallback table ## +# Mirrors the switch in the operator's binding/cndev/library_device.go, used only when +# the driver exports neither cndevGetCardNameStringByDevId nor cndevGetCardNameString. +# The MLU220 variants all report as the family, and anything unlisted reports as "MLU". +_cndevCardNames = { + CNDEV_DEVICE_TYPE_MLU100: "MLU100", + CNDEV_DEVICE_TYPE_MLU270: "MLU270", + CNDEV_DEVICE_TYPE_MLU220_M2: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_EDGE: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_EVB: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_M2i: "MLU220", + CNDEV_DEVICE_TYPE_MLU290: "MLU290", + CNDEV_DEVICE_TYPE_MLU370: "MLU370", + CNDEV_DEVICE_TYPE_MLU365: "MLU365", + CNDEV_DEVICE_TYPE_CE3226: "CE3226", + CNDEV_DEVICE_TYPE_MLU590: "MLU590", + CNDEV_DEVICE_TYPE_MLU585: "MLU585", + CNDEV_DEVICE_TYPE_MLU580: "MLU580", + CNDEV_DEVICE_TYPE_MLU570: "MLU570", +} + + +## string/bytes conversion for ease of use +def convertStrBytes(func): + @wraps(func) + def wrapper(*args, **kwargs): + # encoding a str returns bytes in python 2 and 3 + args = [arg.encode() if isinstance(arg, str) else arg for arg in args] + res = func(*args, **kwargs) + # In python 2, str and bytes are the same + # In python 3, str is unicode and should be decoded. + # Ctypes handles most conversions, this only effects c_char and char arrays. + if isinstance(res, bytes): + if isinstance(res, str): + return res + return res.decode() + return res + + return wrapper + + +def _LoadCndevLibrary(): + """ + Load the library if it isn't loaded already. + """ + global cndevLib + + if cndevLib is None: + # lock to ensure only one caller loads the library + libLoadLock.acquire() + + try: + # ensure the library still isn't loaded + if cndevLib is None: + if sys.platform.startswith("win"): + # Do not support Windows yet. + raise CNDevError(CNDEV_ERROR_LIBRARY_NOT_FOUND) + # Linux path, + # mirroring the search order of the operator's binding/cndev/library.go: + # the bare soname first, so a driver on the loader's search path wins, + # then the Neuware home, where the CNToolkit package installs it. + neuware_home = os.getenv("NEUWARE_HOME") or "/usr/local/neuware" + locs = [ + "libcndev.so", + os.path.join(neuware_home, "lib64", "libcndev.so"), + os.path.join(neuware_home, "lib", "libcndev.so"), + ] + load_error = None + for loc in locs: + try: + cndevLib = CDLL(loc) + break + except OSError as e: + load_error = e + if cndevLib is None: + # Chain the loader's own complaint: a library that is present + # but unloadable -- wrong architecture, a missing dependency, + # no permission -- is otherwise indistinguishable from one + # that is absent, and is_supported() only logs what it caught. + raise CNDevError(CNDEV_ERROR_LIBRARY_NOT_FOUND) from load_error + finally: + # lock is always freed + libLoadLock.release() + + +## C function wrappers ## +def cndevInit(): + _LoadCndevLibrary() + + # Initialize the library + global _libInitialized, _libInitializedException + + # Checking the flag outside the lock lets two concurrent first callers both reach + # the driver, and the one that loses is answered with an already-initialized error + # that is then cached as the library's permanent state, disabling detection until + # the process restarts. The operator's binding uses sync.Once for the same reason, + # so the whole sequence is held here. + with libLoadLock: + if _libInitialized: + if _libInitializedException is not None: + # Re-raise a fresh copy: re-raising the same cached exception object + # appends a traceback frame on every call, and those frames retain the + # caller's locals, leaking memory over time. See gpustack/gpustack#5342. + from ..__utils__ import clone_exception + + raise clone_exception(_libInitializedException) from None + return + + try: + fn = _cndevGetFunctionPointer("cndevInit") + # The header names the parameter "reserved" and the operator passes 0. + # Version negotiation happens per struct, not here. + ret = fn(c_int32(0)) + _cndevCheckReturn(ret) + except Exception as e: + _libInitializedException = e + raise + finally: + _libInitialized = True + + +def cndevRelease(): + global _libInitialized, _libInitializedException + + with libLoadLock: + if not _libInitialized: + return + + # Initialization that never reached the driver left nothing to release. + if _libInitializedException is None: + # Unlike the sibling bindings' shutdown, this reaches the driver: + # cndevRelease frees what cndevInit allocated, as the operator's binding + # does. A driver too old to export it has nothing to free. + try: + fn = _cndevGetFunctionPointer("cndevRelease") + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + else: + # Clear the flags only once the driver has let go. Clearing them + # first would report the library uninitialized while it still + # holds what cndevInit allocated, and the next cndevInit would + # then initialize on top of it. + _cndevCheckReturn(fn()) + + _libInitialized = False + _libInitializedException = None + + +def cndevGetDeviceCount(): + c_card_info = c_cndevCardInfo_t() + c_card_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetDeviceCount") + ret = fn(byref(c_card_info)) + _cndevCheckReturn(ret) + return c_card_info.number + + +def cndevGetDeviceHandleByIndex(index): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByIndex") + ret = fn(index, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +@convertStrBytes +def cndevGetDeviceHandleByUUID(uuid): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByUUID") + ret = fn(uuid, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +@convertStrBytes +def cndevGetDeviceHandleByPciBusId(pciBusId): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByPciBusId") + ret = fn(pciBusId, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +def cndevGetUUID(device): + c_uuid_info = c_cndevUUID_t() + c_uuid_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetUUID") + ret = fn(byref(c_uuid_info), device) + _cndevCheckReturn(ret) + # An empty field is not an identity: prefixing it would yield a bare "MLU-" that + # reads as a valid id and is the same for every card, so the usage join would write + # one card's metrics onto all of them. The caller's contract is that a card whose + # identity cannot be read is skipped, so say so rather than manufacture one. + if not c_uuid_info.uuid: + raise CNDevError(CNDEV_ERROR_NOT_FOUND) + # The driver reports the bare UUID. Every Cambricon tool, and the operator's + # UUID.String(), names the device with the "MLU-" prefix, so the prefix belongs here + # rather than in each caller. + return "MLU-" + c_uuid_info.uuid + + +def cndevGetPCIeInfoV2(device): + c_pcie_info = c_cndevPCIeInfoV2_t() + fn = _cndevGetFunctionPointer("cndevGetPCIeInfoV2") + ret = fn(byref(c_pcie_info), device) + _cndevCheckReturn(ret) + return c_pcie_info + + +def cndevGetPCIeBusId(device): + c_pcie_info = cndevGetPCIeInfoV2(device) + + domain = c_pcie_info.domain + bus = c_pcie_info.bus + dev = c_pcie_info.device + function = c_pcie_info.function + return f"{domain:04x}:{bus:02x}:{dev:02x}.{function:d}" + + +def cndevGetCardName(device): + c_card_name = c_cndevCardName_t() + c_card_name.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetCardName") + ret = fn(byref(c_card_name), device) + _cndevCheckReturn(ret) + return c_card_name + + +@convertStrBytes +def cndevGetCardNameString(cardName): + fn = _cndevGetFunctionPointer("cndevGetCardNameString") + fn.restype = c_char_p + return fn(cardName) + + +@convertStrBytes +def cndevGetCardNameStringByDevId(device): + fn = _cndevGetFunctionPointer("cndevGetCardNameStringByDevId") + fn.restype = c_char_p + return fn(device) + + +def cndevGetCardNameByDevId(device): + """ + Resolve the device's marketing name, whatever the driver offers. + + Not a CNDev entry point: it applies the precedence of the operator's + Device.GetCardName -- the name string by device, then the name enum resolved by the + library, then the enum resolved locally -- so the fallback lives in one place + instead of in every caller. + + Args: + device: + The device handle. + + Returns: + The device name, "MLU" when the driver reports an unknown name enum. + + Raises: + CNDevError: If the driver fails for a reason other than a missing symbol. + + """ + name = None + try: + name = cndevGetCardNameStringByDevId(device) + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + if name: + return name + + c_card_name = cndevGetCardName(device) + + try: + name = cndevGetCardNameString(c_card_name.id) + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + if name: + return name + + return _cndevCardNames.get(c_card_name.id, "MLU") + + +def cndevGetMemoryUsageV2(device): + c_memory_info = c_cndevMemoryInfoV2_t() + fn = _cndevGetFunctionPointer("cndevGetMemoryUsageV2") + ret = fn(byref(c_memory_info), device) + _cndevCheckReturn(ret) + return c_memory_info + + +def cndevGetCardHealthState(device): + c_health_state = c_cndevCardHealthState_t() + c_health_state.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetCardHealthState") + ret = fn(byref(c_health_state), device) + _cndevCheckReturn(ret) + return c_health_state + + +def cndevGetCardHealthStateV2(device): + c_health_state = c_cndevCardHealthStateV2_t() + try: + fn = _cndevGetFunctionPointer("cndevGetCardHealthStateV2") + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + # Mirror the operator's CardHealthStateHandler.V1: an older driver reports the + # same three fields through the V1 struct, so map them into the V2 shape and + # leave the incident report empty rather than failing the whole detection. + c_health_state_v1 = cndevGetCardHealthState(device) + c_health_state.health = c_health_state_v1.health + c_health_state.deviceState = c_health_state_v1.deviceState + c_health_state.driverState = c_health_state_v1.driverState + # V1 carries no overall verdict, and leaving the field at zero would read + # as CNDEV_HEALTH_RESULT_PASS: a false pass on every old driver, for any + # caller that reaches for it instead of the health bit. Derive it. + c_health_state.overallHealth = ( + CNDEV_HEALTH_RESULT_PASS + if c_health_state_v1.health == CNDEV_HEALTH_STATE_DEVICE_GOOD + else CNDEV_HEALTH_RESULT_FAIL + ) + return c_health_state + ret = fn(byref(c_health_state), device) + _cndevCheckReturn(ret) + return c_health_state + + +def cndevGetVersionInfo(device): + c_version_info = c_cndevVersionInfo_t() + c_version_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetVersionInfo") + ret = fn(byref(c_version_info), device) + _cndevCheckReturn(ret) + return c_version_info + + +def cndevGetDeviceUtilizationInfo(device): + c_util_info = c_cndevUtilizationInfo_t() + c_util_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetDeviceUtilizationInfo") + ret = fn(byref(c_util_info), device) + _cndevCheckReturn(ret) + return c_util_info + + +def cndevGetTemperatureInfo(device): + c_temperature_info = c_cndevTemperatureInfo_t() + # cndev.h declares this struct's version as IN, so it is stamped like every other + # versioned struct. The operator's GetTemperatureInfo is the one wrapper that leaves + # it zero, which its own GetVersionInfo comment suggests is an oversight surviving on + # its driver: an unstamped struct is answered with ERROR_UNSUPPORTED_API_VERSION. + # Temperature is a usage field the detector reads under suppression, so that error + # would read as no temperature rather than fail loudly. Confirmed on real hardware at + # checkpoint C2. + c_temperature_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetTemperatureInfo") + ret = fn(byref(c_temperature_info), device) + _cndevCheckReturn(ret) + return c_temperature_info + + +def cndevGetDevicePowerInfo(device): + c_power_info = c_cndevDevicePowerInfo_t() + fn = _cndevGetFunctionPointer("cndevGetDevicePowerInfo") + ret = fn(byref(c_power_info), device) + _cndevCheckReturn(ret) + return c_power_info + + +def cndevGetNUMANodeIdByDevId(device): + c_numa_node_id = c_cndevNUMANodeId_t() + c_numa_node_id.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetNUMANodeIdByDevId") + ret = fn(byref(c_numa_node_id), device) + _cndevCheckReturn(ret) + return c_numa_node_id + + +def cndevGetECCInfo(device): + c_ecc_info = c_cndevECCInfo_t() + c_ecc_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetECCInfo") + ret = fn(byref(c_ecc_info), device) + _cndevCheckReturn(ret) + return c_ecc_info diff --git a/ruff.toml b/ruff.toml index a181dd6..80ced51 100644 --- a/ruff.toml +++ b/ruff.toml @@ -123,6 +123,7 @@ parametrize-names-type = "csv" "gpustack_runtime/deployer/__patches__.py" = ["ALL"] "gpustack_runtime/detector/pyamdsmi/*.py" = ["ALL"] "gpustack_runtime/detector/pyamdgpu/*.py" = ["ALL"] +"gpustack_runtime/detector/pycndev/*.py" = ["ALL"] "gpustack_runtime/detector/pydcmi/*.py" = ["ALL"] "gpustack_runtime/detector/pyhgml/*.py" = ["ALL"] "gpustack_runtime/detector/pyhsa/*.py" = ["ALL"] diff --git a/tests/gpustack_runtime/detector/test_cambricon.py b/tests/gpustack_runtime/detector/test_cambricon.py index 1f8f9d0..2b0aed5 100644 --- a/tests/gpustack_runtime/detector/test_cambricon.py +++ b/tests/gpustack_runtime/detector/test_cambricon.py @@ -1,5 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import cambricon, pycndev +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.cambricon import CambriconDetector @@ -21,3 +31,538 @@ def test_get_topology(): det = CambriconDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pycndev, so the split is provable on a host with no Cambricon driver. # +# --------------------------------------------------------------------------- # + +_HANDLE_OFFSET = 100 +""" +Distance between a card's index and the handle the fake driver hands out. +cndevDevice_t is an int32 handle, i.e. indistinguishable from an index at the +Python level, so the fake offsets it: a detector passing an index where a handle +belongs indexes out of range instead of quietly working. +""" + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MLU590-M9" + memory_total: int = 49152 + memory_used: int = 1024 + driver_version: tuple[int, int, int] = (5, 10, 22) + bus_id: str = "0000:1f:00.0" + numa_node: int = 0 + # The header's `health` carries the same 0/1 convention as the device-state + # enumerators, and the operator's verdict is a bare `Health == 0`. + health: int = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + core_utilization: int = 42 + temperature: int = 55 + power_usage: int = 150 + failing: tuple[str, ...] = () + """ + Entry points that raise for this card, so a faulty card is reproducible. + """ + + +@dataclass +class _MemoryInfo: + """ + A c_cndevMemoryInfoV2_t stand-in. The header calls the unit MB; the operator, + cnmon and this repo's Ascend detector all treat it as MiB. + """ + + physicalMemoryTotal: int # noqa: N815 + physicalMemoryUsed: int # noqa: N815 + + +@dataclass +class _VersionInfo: + """ + A c_cndevVersionInfo_t stand-in, of which only the driver triplet is read. + """ + + driverMajorVersion: int # noqa: N815 + driverMinorVersion: int # noqa: N815 + driverBuildVersion: int # noqa: N815 + + +@dataclass +class _NUMANodeId: + """ + A c_cndevNUMANodeId_t stand-in. + """ + + nodeId: int # noqa: N815 + + +@dataclass +class _CardHealthState: + """ + A c_cndevCardHealthStateV2_t stand-in, of which the detector reads the + card-wide health bit alone, as the operator does. + """ + + health: int + deviceState: int # noqa: N815 + + +@dataclass +class _UtilizationInfo: + """ + A c_cndevUtilizationInfo_t stand-in. + """ + + averageCoreUtilization: int # noqa: N815 + + +@dataclass +class _TemperatureInfo: + """ + A c_cndevTemperatureInfo_t stand-in, in degrees Celsius. + """ + + chip: int + + +@dataclass +class _PowerInfo: + """ + A c_cndevDevicePowerInfo_t stand-in, in Watts. + """ + + usage: int + + +@dataclass +class _FakeCNDev: + """ + A stand-in for the pycndev binding, recording every call the detector makes. + + The error type, the error codes and the health enumerators are the real + module's, so a fake drifting from the binding's contract fails here rather + than on hardware. Entry points are dispatched by name, as the sibling vendors' + fakes do, which keeps the driver's camelCase out of the handlers' own names. + + cndevGetECCInfo is deliberately absent: the operator's Cambricon health + verdict reads the card health state and nothing else, so an ECC read + reintroduced here fails loudly instead of silently costing a driver call. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + CNDevError = pycndev.CNDevError + CNDEV_HEALTH_STATE_DEVICE_GOOD = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + + def __getattr__(self, name: str): + handler = { + "cndevInit": self._init, + "cndevGetDeviceCount": self._get_device_count, + "cndevGetDeviceHandleByIndex": self._get_device_handle_by_index, + "cndevGetUUID": self._get_uuid, + "cndevGetCardNameByDevId": self._get_card_name_by_dev_id, + "cndevGetMemoryUsageV2": self._get_memory_usage_v2, + "cndevGetVersionInfo": self._get_version_info, + "cndevGetPCIeBusId": self._get_pcie_bus_id, + "cndevGetNUMANodeIdByDevId": self._get_numa_node_id_by_dev_id, + "cndevGetCardHealthStateV2": self._get_card_health_state_v2, + "cndevGetDeviceUtilizationInfo": self._get_device_utilization_info, + "cndevGetTemperatureInfo": self._get_temperature_info, + "cndevGetDevicePowerInfo": self._get_device_power_info, + }.get(name) + if handler is None: + msg = f"module pycndev has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _card(self, handle: int, name: str) -> _Card: + card = self.cards[handle - _HANDLE_OFFSET] + if name in card.failing: + raise pycndev.CNDevError(pycndev.CNDEV_ERROR_UNKNOWN) + return card + + def _init(self) -> None: + pass + + def _get_device_count(self) -> int: + return len(self.cards) + + def _get_device_handle_by_index(self, index: int) -> int: + return index + _HANDLE_OFFSET + + def _get_uuid(self, handle: int) -> str: + # The binding prefixes the driver's bare UUID, as every Cambricon tool does. + return "MLU-" + self._card(handle, "cndevGetUUID").uuid + + def _get_card_name_by_dev_id(self, handle: int) -> str: + return self._card(handle, "cndevGetCardNameByDevId").name + + def _get_memory_usage_v2(self, handle: int) -> _MemoryInfo: + card = self._card(handle, "cndevGetMemoryUsageV2") + return _MemoryInfo( + physicalMemoryTotal=card.memory_total, + physicalMemoryUsed=card.memory_used, + ) + + def _get_version_info(self, handle: int) -> _VersionInfo: + card = self._card(handle, "cndevGetVersionInfo") + major, minor, build = card.driver_version + return _VersionInfo( + driverMajorVersion=major, + driverMinorVersion=minor, + driverBuildVersion=build, + ) + + def _get_pcie_bus_id(self, handle: int) -> str: + return self._card(handle, "cndevGetPCIeBusId").bus_id + + def _get_numa_node_id_by_dev_id(self, handle: int) -> _NUMANodeId: + card = self._card(handle, "cndevGetNUMANodeIdByDevId") + return _NUMANodeId(nodeId=card.numa_node) + + def _get_card_health_state_v2(self, handle: int) -> _CardHealthState: + card = self._card(handle, "cndevGetCardHealthStateV2") + return _CardHealthState(health=card.health, deviceState=card.health) + + def _get_device_utilization_info(self, handle: int) -> _UtilizationInfo: + card = self._card(handle, "cndevGetDeviceUtilizationInfo") + return _UtilizationInfo(averageCoreUtilization=card.core_utilization) + + def _get_temperature_info(self, handle: int) -> _TemperatureInfo: + card = self._card(handle, "cndevGetTemperatureInfo") + return _TemperatureInfo(chip=card.temperature) + + def _get_device_power_info(self, handle: int) -> _PowerInfo: + card = self._card(handle, "cndevGetDevicePowerInfo") + return _PowerInfo(usage=card.power_usage) + + +_USAGE_CALLS = ( + "cndevGetDeviceUtilizationInfo", + "cndevGetTemperatureInfo", + "cndevGetDevicePowerInfo", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: cndevGetMemoryUsageV2 +reports the memory *total* the inventory needs alongside the used amount, so +both queries read it, as the operator's DetectAccelerator and MonitorAccelerator +both do. +""" + +_MEMORY_UTILIZATION = 2.08 +""" +A default card's memory utilization: 1024 MiB used of 49152 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch, tmp_path): + """ + Build a Cambricon detector talking to a fake driver reporting the given cards. + """ + + def _install( + *cards: _Card, + neuware_version: str | None = "Neuware Version: 1.2.3\n", + ) -> tuple[CambriconDetector, _FakeCNDev]: + fake = _FakeCNDev(cards=list(cards)) + monkeypatch.setattr(cambricon, "pycndev", fake) + + # The Neuware version is a file read, not a driver call, so it is faked + # by pointing the module's path constant at a temporary file. + version_path = tmp_path / "version.txt" + if neuware_version is not None: + version_path.write_text(neuware_version) + monkeypatch.setattr(cambricon, "_NEUWARE_VERSION_PATH", version_path) + + det = CambriconDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: inventory only, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="0123456789ab")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.CAMBRICON + assert dev.index == 0 + assert dev.name == "MLU590-M9" + assert dev.uuid == "MLU-0123456789ab" + # The operator formats major.minor only; the build number is free precision + # from the same query, and it is the digit a driver bug is diagnosed by. + assert dev.driver_version == "5.10.22" + assert dev.runtime_version == "1.2" + assert dev.runtime_version_original == "1.2.3" + # The driver reports MB per the header and MiB in fact, so no conversion is + # applied: converting would under-report ~4.8% against the operator. + assert dev.memory == 49152 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:1f:00.0" + assert dev.appendix["numa"] == "0" + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + # No power call is made, so the power limit stays unset. + assert dev.power is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="0123456789ab")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + assert "cndevGetMemoryUsageV2" in fake.calls + assert "cndevGetVersionInfo" in fake.calls + + +def test_detect_info_enumerates_every_card(detector): + det, _ = detector(_Card(uuid="card-0"), _Card(uuid="card-1")) + + devices = det.detect_info() + + assert [dev.uuid for dev in devices] == ["MLU-card-0", "MLU-card-1"] + assert [dev.index for dev in devices] == [0, 1] + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector(_Card(uuid="card-0")) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +def test_detect_info_without_the_neuware_version_file(detector): + det, _ = detector(_Card(uuid="card-0"), neuware_version=None) + + dev = det.detect_info()[0] + + assert dev.runtime_version is None + assert dev.runtime_version_original is None + + +def test_detect_info_ignores_an_unversioned_neuware_file(detector): + det, _ = detector(_Card(uuid="card-0"), neuware_version="unknown\n") + + dev = det.detect_info()[0] + + assert dev.runtime_version_original is None + + +# --------------------------------------------------------------------------- # +# A faulty card is skipped, not fatal. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "failing", + ["cndevGetUUID", "cndevGetMemoryUsageV2", "cndevGetCardNameByDevId"], +) +def test_detect_info_skips_a_faulty_card(detector, failing): + det, _ = detector( + _Card(uuid="broken", failing=(failing,)), + _Card(uuid="healthy"), + ) + + devices = det.detect_info() + + # The operator continues past a card whose required reads fail. Without that, + # one faulty card makes the whole vendor report zero devices. + assert [dev.uuid for dev in devices] == ["MLU-healthy"] + # Skipping does not renumber: the index is the one the driver enumerated the + # card at, as Device.index promises. + assert devices[0].index == 1 + + +def test_detect_info_fails_when_every_card_is_skipped(detector): + # Skipping one card of several is the graceful degradation this vendor is + # allowed. Every card failing the same read is systemic -- a driver not + # exporting a call the loop needs -- and an empty inventory would be + # indistinguishable from a host that has no MLU at all. + det, _ = detector( + _Card(uuid="broken-0", failing=("cndevGetPCIeBusId",)), + _Card(uuid="broken-1", failing=("cndevGetPCIeBusId",)), + ) + + with pytest.raises(pycndev.CNDevError): + det.detect_info() + + +def test_detect_usage_fails_when_every_card_is_skipped(detector): + det, _ = detector( + _Card(uuid="broken-0", failing=("cndevGetMemoryUsageV2",)), + _Card(uuid="broken-1", failing=("cndevGetMemoryUsageV2",)), + ) + + with pytest.raises(pycndev.CNDevError): + det.detect_usage() + + +def test_detect_info_tolerates_an_unreadable_optional_field(detector): + det, _ = detector( + _Card( + uuid="card-0", + failing=("cndevGetVersionInfo", "cndevGetNUMANodeIdByDevId"), + ), + ) + + dev = det.detect_info()[0] + + # The optional reads are suppressed, so the card is still reported. + assert dev.uuid == "MLU-card-0" + assert dev.driver_version is None + assert "numa" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="card-0"), + _Card( + uuid="card-1", + memory_used=2048, + core_utilization=7, + temperature=61, + power_usage=200, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["MLU-card-0"].cores_utilization == 42 + assert by_uuid["MLU-card-0"].memory_used == 1024 + assert by_uuid["MLU-card-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["MLU-card-0"].temperature == 55 + assert by_uuid["MLU-card-0"].power_used == 150 + assert by_uuid["MLU-card-1"].cores_utilization == 7 + assert by_uuid["MLU-card-1"].memory_used == 2048 + assert by_uuid["MLU-card-1"].temperature == 61 + assert by_uuid["MLU-card-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["MLU-card-0"].index == 0 + assert by_uuid["MLU-card-0"].memory == 49152 + assert by_uuid["MLU-card-0"].name == "MLU590-M9" + assert by_uuid["MLU-card-0"].driver_version == "5.10.22" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="card-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["MLU-card-0"] + assert devices[0].name == "MLU590-M9" + assert devices[0].memory == 49152 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="card-0")) + + dev = det.detect()[0] + + assert dev.name == "MLU590-M9" + assert dev.memory == 49152 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +def test_detect_usage_skips_a_faulty_card(detector): + det, _ = detector( + _Card(uuid="broken", failing=("cndevGetMemoryUsageV2",)), + _Card(uuid="healthy"), + ) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["MLU-healthy"] + assert devices[0].memory_used == 1024 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="card-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reads_no_health_state_by_default(detector): + det, fake = detector(_Card(uuid="card-0")) + + det.detect() + + # GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults to true, so a default run + # pays for no health call at all -- a deliberate divergence from the + # operator, which reads the card health unconditionally. + assert "cndevGetCardHealthStateV2" not in fake.calls + + +def test_detect_reports_a_card_in_problem(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector( + _Card( + uuid="card-0", + health=pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM, + ), + ) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_tolerates_an_unreadable_health_state(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector( + _Card(uuid="card-0", failing=("cndevGetCardHealthStateV2",)), + ) + + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY diff --git a/tests/gpustack_runtime/detector/test_pycndev.py b/tests/gpustack_runtime/detector/test_pycndev.py new file mode 100644 index 0000000..af8922d --- /dev/null +++ b/tests/gpustack_runtime/detector/test_pycndev.py @@ -0,0 +1,926 @@ +from __future__ import annotations as __future_annotations__ + +import importlib.util +import sys +import threading +from ctypes import sizeof +from unittest import mock + +import pytest + +from gpustack_runtime.detector import pycndev + +MODULE_NAME = "gpustack_runtime.detector.pycndev" + +# Expected layout of every bound struct: total size, then (field, offset, size) in the +# header's own field order. +# +# Derived from the operator's binding/cndev/cndev.h -- the header of record -- by +# compiling a probe that prints sizeof() and offsetof() for each field, so the table is +# independent of the ctypes definitions it checks. That independence is the point: no +# Cambricon hardware exists in this suite, the driver fills these buffers by offset, and +# a wrong width or a missing field would corrupt memory rather than fail a test. +# +# Every struct here is plain fixed-width scalars and arrays -- no bitfields, no packing, +# no pointers -- so these numbers hold on Linux x86_64 and arm64 alike. They are not +# specific to the host that ran the probe. +EXPECTED_LAYOUTS = { + "c_cndevCardInfo_t": ( + 8, + [("version", 0, 4), ("number", 4, 4)], + ), + "c_cndevUUID_t": ( + # uuid is 37 bytes at offset 4, so 7 bytes of padding precede the 8-aligned + # ncsUUID64 at 48. + 56, + [("version", 0, 4), ("uuid", 4, 37), ("ncsUUID64", 48, 8)], + ), + "c_cndevMemoryInfoV2_t": ( + 176, + [ + ("physicalMemoryTotal", 0, 8), + ("physicalMemoryUsed", 8, 8), + ("reservedMemory", 16, 8), + ("virtualMemoryTotal", 24, 8), + ("virtualMemoryUsed", 32, 8), + ("globalMemory", 40, 8), + ("reserved", 48, 128), + ], + ), + "c_cndevVersionInfo_t": ( + 28, + [ + ("version", 0, 4), + ("mcuMajorVersion", 4, 4), + ("mcuMinorVersion", 8, 4), + ("mcuBuildVersion", 12, 4), + ("driverMajorVersion", 16, 4), + ("driverMinorVersion", 20, 4), + ("driverBuildVersion", 24, 4), + ], + ), + "c_cndevECCInfo_t": ( + # 4 bytes of padding follow the version, so the first counter is 8-aligned. + 72, + [ + ("version", 0, 4), + ("oneBitError", 8, 8), + ("multipleOneError", 16, 8), + ("multipleError", 24, 8), + ("multipleMultipleError", 32, 8), + ("correctedError", 40, 8), + ("uncorrectedError", 48, 8), + ("totalError", 56, 8), + ("addressForbiddenError", 64, 8), + ], + ), + "c_cndevDevicePowerInfo_t": ( + 84, + [ + ("usage", 0, 4), + ("cap", 4, 4), + ("machine", 8, 4), + ("tdp", 12, 4), + ("maxPower", 16, 4), + ("reserved", 20, 64), + ], + ), + "c_cndevTemperatureInfo_t": ( + 148, + [ + ("version", 0, 4), + ("board", 4, 4), + ("cluster", 8, 80), + ("memoryDie", 88, 32), + ("chip", 120, 4), + ("airInlet", 124, 4), + ("airOutlet", 128, 4), + ("memory", 132, 4), + ("videoInput", 136, 4), + ("cpu", 140, 4), + ("isp", 144, 4), + ], + ), + "c_cndevUtilizationInfo_t": ( + 328, + [ + ("version", 0, 4), + ("averageCoreUtilization", 4, 4), + ("coreUtilization", 8, 320), + ], + ), + "c_cndevCardName_t": ( + 8, + [("version", 0, 4), ("id", 4, 4)], + ), + "c_cndevPCIeInfoV2_t": ( + 64, + [ + ("subsystemId", 0, 4), + ("deviceId", 4, 4), + ("vendor", 8, 2), + ("subsystemVendor", 10, 2), + ("domain", 12, 4), + ("bus", 16, 4), + ("device", 20, 4), + ("function", 24, 4), + ("moduleId", 28, 2), + ("slotId", 30, 2), + ("reserved", 32, 32), + ], + ), + "c_cndevCardHealthState_t": ( + 16, + [ + ("version", 0, 4), + ("health", 4, 4), + ("deviceState", 8, 4), + ("driverState", 12, 4), + ], + ), + "c_cndevDiagErrorDetail_t": ( + 532, + [ + ("msg", 0, 512), + ("device_id", 512, 4), + ("bdf", 516, 4), + ("code", 520, 4), + ("category", 524, 4), + ("severity", 528, 4), + ], + ), + "c_cndevIncidentInfo_t": ( + 540, + [("system", 0, 4), ("health", 4, 4), ("error", 8, 532)], + ), + "c_cndevCardHealthStateV2_t": ( + # 64 incident slots, which the driver fills in place: a short struct here is a + # buffer overrun, not a missing field. + 34612, + [ + ("health", 0, 4), + ("deviceState", 4, 4), + ("driverState", 8, 4), + ("overallHealth", 12, 4), + ("incident_count", 16, 4), + ("incidents", 20, 34560), + ("reserved", 34580, 32), + ], + ), + "c_cndevNUMANodeId_t": ( + 8, + [("version", 0, 4), ("nodeId", 4, 4)], + ), +} + +# Constants pinned to the literals in the operator's binding/cndev/cndev.h -- the header +# of record -- rather than to the module's own definitions. +# +# Asserting a constant against itself proves nothing: CNDEV_VERSION_6 defined as 7 would +# leave this suite green while every versioned call on an MLU host was answered with +# CNDEV_ERROR_UNSUPPORTED_API_VERSION. No Cambricon hardware exists here, so a mistyped +# constant has to fail below or it fails at a customer. +HEADER_API_VERSION = 6 + +# cndevRet_enum, in the header's own order. +HEADER_ERROR_CODES = { + "CNDEV_SUCCESS": 0, + "CNDEV_ERROR_NO_DRIVER": 1, + "CNDEV_ERROR_LOW_DRIVER_VERSION": 2, + "CNDEV_ERROR_UNSUPPORTED_API_VERSION": 3, + "CNDEV_ERROR_UNINITIALIZED": 4, + "CNDEV_ERROR_INVALID_ARGUMENT": 5, + "CNDEV_ERROR_INVALID_DEVICE_ID": 6, + "CNDEV_ERROR_UNKNOWN": 7, + "CNDEV_ERROR_MALLOC": 8, + "CNDEV_ERROR_INSUFFICIENT_SPACE": 9, + "CNDEV_ERROR_NOT_SUPPORTED": 10, + "CNDEV_ERROR_INVALID_LINK": 11, + "CNDEV_ERROR_NO_DEVICES": 12, + "CNDEV_ERROR_NO_PERMISSION": 13, + "CNDEV_ERROR_NOT_FOUND": 14, + "CNDEV_ERROR_IN_USE": 15, + "CNDEV_ERROR_DUPLICATE": 16, + "CNDEV_ERROR_TIMEOUT": 17, + "CNDEV_ERROR_IN_PROBLEM": 18, +} + +# cndevNameEnum_t, limited to the types the card-name fallback table answers for: a wrong +# value there names the wrong card on a driver that exports neither name string. +HEADER_DEVICE_TYPES = { + "CNDEV_DEVICE_TYPE_MLU100": 0, + "CNDEV_DEVICE_TYPE_MLU270": 1, + "CNDEV_DEVICE_TYPE_MLU220_M2": 16, + "CNDEV_DEVICE_TYPE_MLU220_EDGE": 17, + "CNDEV_DEVICE_TYPE_MLU220_EVB": 18, + "CNDEV_DEVICE_TYPE_MLU220_M2i": 19, + "CNDEV_DEVICE_TYPE_MLU290": 20, + "CNDEV_DEVICE_TYPE_MLU370": 23, + "CNDEV_DEVICE_TYPE_MLU365": 24, + "CNDEV_DEVICE_TYPE_CE3226": 25, + "CNDEV_DEVICE_TYPE_MLU590": 26, + "CNDEV_DEVICE_TYPE_MLU585": 27, + "CNDEV_DEVICE_TYPE_MLU580": 30, + "CNDEV_DEVICE_TYPE_MLU570": 31, +} + +# What the fake library reports, so an assertion names a value rather than a literal. +FAKE_DEVICE_COUNT = 4 +FAKE_HANDLE_BASE = 100 +FAKE_UUID = "d4e5f60718293a4b" +FAKE_CARD_NAME = "MLU590-M9" +FAKE_MEMORY_TOTAL = 48000 +FAKE_MEMORY_USED = 1024 +FAKE_DRIVER_VERSION = (6, 10, 3) +FAKE_CORE_UTILIZATION = 37 +FAKE_BOARD_TEMPERATURE = 45 +FAKE_POWER_USAGE = 110 +FAKE_POWER_CAP = 250 +FAKE_NUMA_NODE = 1 +FAKE_CORRECTED_ERRORS = 2 +FAKE_PCIE_BUS_ID = "0000:3b:00.0" + +# A device handle the fake library answers for. +DEVICE = FAKE_HANDLE_BASE + + +def _identity(obj): + """ + Stand in for ctypes.byref. + + The fake library's entry points are plain Python callables, so a real byref() would + hand them a CArgObject that cannot be read or written from Python. Passing the + struct itself keeps what the wrapper sent -- and what it expects back -- visible. + """ + return obj + + +def _default_handlers(): + """ + Build one handler per libcndev.so entry point the binding may call. + + Each handler fills the out-struct the way a healthy MLU590 card would and returns + CNDEV_SUCCESS. + """ + + def init(reserved): + assert reserved is not None + return pycndev.CNDEV_SUCCESS + + def release(): + return pycndev.CNDEV_SUCCESS + + def get_device_count(card_info): + card_info.number = FAKE_DEVICE_COUNT + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_index(index, handle): + handle.value = FAKE_HANDLE_BASE + index + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_uuid(uuid, handle): + assert isinstance(uuid, bytes) + handle.value = FAKE_HANDLE_BASE + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_pci_bus_id(pci_bus_id, handle): + assert isinstance(pci_bus_id, bytes) + handle.value = FAKE_HANDLE_BASE + return pycndev.CNDEV_SUCCESS + + def get_uuid(uuid_info, device): + assert device == DEVICE + uuid_info.uuid = FAKE_UUID + return pycndev.CNDEV_SUCCESS + + def get_pcie_info_v2(pcie_info, device): + assert device == DEVICE + pcie_info.domain = 0x0000 + pcie_info.bus = 0x3B + pcie_info.device = 0x00 + pcie_info.function = 0 + return pycndev.CNDEV_SUCCESS + + def get_card_name(card_name, device): + assert device == DEVICE + card_name.id = pycndev.CNDEV_DEVICE_TYPE_MLU590 + return pycndev.CNDEV_SUCCESS + + def get_card_name_string(card_name_id): + assert card_name_id == pycndev.CNDEV_DEVICE_TYPE_MLU590 + return b"MLU590" + + def get_card_name_string_by_dev_id(device): + assert device == DEVICE + return FAKE_CARD_NAME.encode() + + def get_memory_usage_v2(memory_info, device): + assert device == DEVICE + memory_info.physicalMemoryTotal = FAKE_MEMORY_TOTAL + memory_info.physicalMemoryUsed = FAKE_MEMORY_USED + return pycndev.CNDEV_SUCCESS + + def get_card_health_state(health_state, device): + assert device == DEVICE + health_state.health = 1 + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + return pycndev.CNDEV_SUCCESS + + def get_card_health_state_v2(health_state, device): + assert device == DEVICE + health_state.health = 1 + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + health_state.overallHealth = pycndev.CNDEV_HEALTH_RESULT_PASS + return pycndev.CNDEV_SUCCESS + + def get_version_info(version_info, device): + assert device == DEVICE + major, minor, build = FAKE_DRIVER_VERSION + version_info.driverMajorVersion = major + version_info.driverMinorVersion = minor + version_info.driverBuildVersion = build + return pycndev.CNDEV_SUCCESS + + def get_device_utilization_info(util_info, device): + assert device == DEVICE + util_info.averageCoreUtilization = FAKE_CORE_UTILIZATION + return pycndev.CNDEV_SUCCESS + + def get_temperature_info(temp_info, device): + assert device == DEVICE + temp_info.board = FAKE_BOARD_TEMPERATURE + temp_info.chip = FAKE_BOARD_TEMPERATURE + return pycndev.CNDEV_SUCCESS + + def get_device_power_info(power_info, device): + assert device == DEVICE + power_info.usage = FAKE_POWER_USAGE + power_info.cap = FAKE_POWER_CAP + return pycndev.CNDEV_SUCCESS + + def get_numa_node_id(numa_node_id, device): + assert device == DEVICE + numa_node_id.nodeId = FAKE_NUMA_NODE + return pycndev.CNDEV_SUCCESS + + def get_ecc_info(ecc_info, device): + assert device == DEVICE + ecc_info.correctedError = FAKE_CORRECTED_ERRORS + return pycndev.CNDEV_SUCCESS + + return { + "cndevInit": init, + "cndevRelease": release, + "cndevGetDeviceCount": get_device_count, + "cndevGetDeviceHandleByIndex": get_device_handle_by_index, + "cndevGetDeviceHandleByUUID": get_device_handle_by_uuid, + "cndevGetDeviceHandleByPciBusId": get_device_handle_by_pci_bus_id, + "cndevGetUUID": get_uuid, + "cndevGetPCIeInfoV2": get_pcie_info_v2, + "cndevGetCardName": get_card_name, + "cndevGetCardNameString": get_card_name_string, + "cndevGetCardNameStringByDevId": get_card_name_string_by_dev_id, + "cndevGetMemoryUsageV2": get_memory_usage_v2, + "cndevGetCardHealthState": get_card_health_state, + "cndevGetCardHealthStateV2": get_card_health_state_v2, + "cndevGetVersionInfo": get_version_info, + "cndevGetDeviceUtilizationInfo": get_device_utilization_info, + "cndevGetTemperatureInfo": get_temperature_info, + "cndevGetDevicePowerInfo": get_device_power_info, + "cndevGetNUMANodeIdByDevId": get_numa_node_id, + "cndevGetECCInfo": get_ecc_info, + } + + +class FakeLibrary: + """ + A stand-in for libcndev.so that records what the binding asked of it. + + A symbol outside its handler set raises AttributeError, which is how a real CDLL + reports a driver too old to export a call. + """ + + def __init__(self, handlers: dict): + self.handlers = handlers + self.symbols: list[str] = [] + self.calls: list[tuple] = [] + + def __getattr__(self, name: str): + handler = self.handlers.get(name) + if handler is None: + raise AttributeError(name) + + self.symbols.append(name) + + def entry_point(*args): + self.calls.append((name, args)) + return handler(*args) + + return entry_point + + +def install_fake_library(monkeypatch, missing: tuple[str, ...] = ()) -> FakeLibrary: + """ + Put a FakeLibrary in place of the loaded libcndev.so, as if cndevInit() had run. + + Args: + monkeypatch: + The pytest monkeypatch fixture, which restores every global afterwards. + missing: + Symbols to withhold, standing in for a driver that does not export them. + + Returns: + The installed FakeLibrary. + + """ + handlers = { + name: handler + for name, handler in _default_handlers().items() + if name not in missing + } + lib = FakeLibrary(handlers) + + monkeypatch.setattr(pycndev, "cndevLib", lib) + monkeypatch.setattr(pycndev, "_libInitialized", True) + monkeypatch.setattr(pycndev, "_libInitializedException", None) + monkeypatch.setattr(pycndev, "_cndevGetFunctionPointer_cache", {}) + monkeypatch.setattr(pycndev, "byref", _identity) + + return lib + + +@pytest.mark.parametrize("name", list(EXPECTED_LAYOUTS)) +def test_struct_layout_matches_header(name): + expected_size, expected_fields = EXPECTED_LAYOUTS[name] + struct = getattr(pycndev, name) + + assert sizeof(struct) == expected_size + + # Field names as well as offsets: an extra field tucked into trailing padding leaves + # every offset and the total size untouched. + assert [field[0] for field in struct._fields_] == [ + field[0] for field in expected_fields + ] + + for field_name, offset, size in expected_fields: + descriptor = getattr(struct, field_name) + assert descriptor.offset == offset, field_name + assert descriptor.size == size, field_name + + +def test_api_version_matches_the_header(): + assert pycndev.CNDEV_VERSION_6 == HEADER_API_VERSION + + +@pytest.mark.parametrize("name, value", list(HEADER_ERROR_CODES.items())) +def test_error_code_matches_the_header(name, value): + assert getattr(pycndev, name) == value + + +@pytest.mark.parametrize("name, value", list(HEADER_DEVICE_TYPES.items())) +def test_device_type_matches_the_header(name, value): + assert getattr(pycndev, name) == value + + +def test_import_makes_no_library_call(monkeypatch): + """ + Importing the module must not reach for libcndev.so. + + The detector package is imported on every host, and all but a few have no Cambricon + driver at all. + """ + # Execute a second, privately named copy: the module registers its error subclasses + # on sys.modules[__name__], so re-executing it under the real name would rebind the + # classes the rest of this file raises and compares. + probe_name = f"{MODULE_NAME}_import_probe" + spec = importlib.util.spec_from_file_location(probe_name, pycndev.__file__) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, probe_name, module) + + with mock.patch( + "ctypes.CDLL", + side_effect=AssertionError("the library was loaded at import time"), + ): + spec.loader.exec_module(module) + + assert module.cndevLib is None + + +def test_missing_library_raises_cndev_error(monkeypatch): + """ + A host without libcndev.so must fail as CNDevError, not as a bare OSError. + + The detector catches the binding's own error type; an OSError escaping from here + would take the whole detection down instead of skipping the vendor. + """ + monkeypatch.setattr(pycndev, "cndevLib", None) + monkeypatch.setattr(pycndev, "_libInitialized", False) + monkeypatch.setattr(pycndev, "_libInitializedException", None) + monkeypatch.setattr( + pycndev, + "CDLL", + mock.Mock(side_effect=OSError("cannot open shared object file")), + ) + + with pytest.raises(pycndev.CNDevError) as excinfo: + pycndev.cndevInit() + + assert excinfo.value == pycndev.CNDEV_ERROR_LIBRARY_NOT_FOUND + assert not isinstance(excinfo.value, OSError) + assert str(excinfo.value) == "Library Not Found" + + +def test_uninitialized_call_raises_cndev_error(monkeypatch): + monkeypatch.setattr(pycndev, "cndevLib", None) + monkeypatch.setattr(pycndev, "_cndevGetFunctionPointer_cache", {}) + + with pytest.raises(pycndev.CNDevError) as excinfo: + pycndev.cndevGetDeviceCount() + + assert excinfo.value == pycndev.CNDEV_ERROR_UNINITIALIZED + + +def test_error_maps_code_to_subclass(): + err = pycndev.CNDevError(pycndev.CNDEV_ERROR_NOT_SUPPORTED) + + assert isinstance(err, pycndev.CNDevError_NotSupported) + assert err == pycndev.CNDEV_ERROR_NOT_SUPPORTED + assert err == pycndev.CNDevError(pycndev.CNDEV_ERROR_NOT_SUPPORTED) + assert err != pycndev.CNDevError(pycndev.CNDEV_ERROR_NO_DRIVER) + assert str(err) == "Not Supported" + + assert ( + pycndev.cndevExceptionClass(pycndev.CNDEV_ERROR_NO_DRIVER) + is pycndev.CNDevError_NoDriver + ) + + +def test_error_renders_an_unknown_code(): + assert str(pycndev.CNDevError(-12345)) == "Unknown CNDev Error -12345" + + +# One entry per wrapper that maps 1:1 onto a C entry point, so a wrapper reaching for +# the wrong symbol fails here rather than on a Cambricon host. +DIRECT_CALLS = [ + ("cndevGetDeviceCount", lambda: pycndev.cndevGetDeviceCount()), + ("cndevGetDeviceHandleByIndex", lambda: pycndev.cndevGetDeviceHandleByIndex(0)), + ("cndevGetDeviceHandleByUUID", lambda: pycndev.cndevGetDeviceHandleByUUID("MLU-x")), + ( + "cndevGetDeviceHandleByPciBusId", + lambda: pycndev.cndevGetDeviceHandleByPciBusId(FAKE_PCIE_BUS_ID), + ), + ("cndevGetUUID", lambda: pycndev.cndevGetUUID(DEVICE)), + ("cndevGetPCIeInfoV2", lambda: pycndev.cndevGetPCIeInfoV2(DEVICE)), + ("cndevGetCardName", lambda: pycndev.cndevGetCardName(DEVICE)), + ( + "cndevGetCardNameString", + lambda: pycndev.cndevGetCardNameString(pycndev.CNDEV_DEVICE_TYPE_MLU590), + ), + ( + "cndevGetCardNameStringByDevId", + lambda: pycndev.cndevGetCardNameStringByDevId(DEVICE), + ), + ("cndevGetMemoryUsageV2", lambda: pycndev.cndevGetMemoryUsageV2(DEVICE)), + ("cndevGetCardHealthState", lambda: pycndev.cndevGetCardHealthState(DEVICE)), + ("cndevGetCardHealthStateV2", lambda: pycndev.cndevGetCardHealthStateV2(DEVICE)), + ("cndevGetVersionInfo", lambda: pycndev.cndevGetVersionInfo(DEVICE)), + ( + "cndevGetDeviceUtilizationInfo", + lambda: pycndev.cndevGetDeviceUtilizationInfo(DEVICE), + ), + ("cndevGetTemperatureInfo", lambda: pycndev.cndevGetTemperatureInfo(DEVICE)), + ("cndevGetDevicePowerInfo", lambda: pycndev.cndevGetDevicePowerInfo(DEVICE)), + ("cndevGetNUMANodeIdByDevId", lambda: pycndev.cndevGetNUMANodeIdByDevId(DEVICE)), + ("cndevGetECCInfo", lambda: pycndev.cndevGetECCInfo(DEVICE)), +] + +# The wrappers whose out-struct carries the header's IN version field. cndev rejects a +# versioned struct that does not declare which layout the caller speaks, so an unstamped +# one is answered with CNDEV_ERROR_UNSUPPORTED_API_VERSION rather than data. +VERSIONED_CALLS = [ + entry + for entry in DIRECT_CALLS + if entry[0] + in { + "cndevGetDeviceCount", + "cndevGetUUID", + "cndevGetCardName", + "cndevGetCardHealthState", + "cndevGetVersionInfo", + "cndevGetDeviceUtilizationInfo", + "cndevGetTemperatureInfo", + "cndevGetNUMANodeIdByDevId", + "cndevGetECCInfo", + } +] + + +@pytest.mark.parametrize("symbol, call", DIRECT_CALLS) +def test_wrapper_calls_expected_symbol(monkeypatch, symbol, call): + lib = install_fake_library(monkeypatch) + + call() + + assert [name for name, _ in lib.calls] == [symbol] + + +@pytest.mark.parametrize("symbol, call", VERSIONED_CALLS) +def test_versioned_call_declares_the_api_version(monkeypatch, symbol, call): + lib = install_fake_library(monkeypatch) + + call() + + _, args = lib.calls[-1] + assert args[0].version == HEADER_API_VERSION, symbol + + +@pytest.mark.parametrize("symbol, call", DIRECT_CALLS) +def test_wrapper_raises_on_driver_failure(monkeypatch, symbol, call): + if symbol in ("cndevGetCardNameString", "cndevGetCardNameStringByDevId"): + pytest.skip("returns a string, not a cndevRet_t") + + lib = install_fake_library(monkeypatch) + lib.handlers[symbol] = lambda *_: pycndev.CNDEV_ERROR_NOT_SUPPORTED + + with pytest.raises(pycndev.CNDevError_NotSupported): + call() + + +def test_get_device_count_returns_the_reported_number(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetDeviceCount() == FAKE_DEVICE_COUNT + + +def test_get_device_handle_by_index_returns_the_handle(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetDeviceHandleByIndex(2) == FAKE_HANDLE_BASE + 2 + + +def test_get_uuid_prefixes_the_manufacturer_tag(monkeypatch): + """ + The UUID must read the way every Cambricon tool and the operator report it. + """ + install_fake_library(monkeypatch) + + assert pycndev.cndevGetUUID(DEVICE) == "MLU-" + FAKE_UUID + + +def test_get_uuid_rejects_an_empty_field(monkeypatch): + """ + A driver answering success with an empty UUID has reported no identity. + + Prefixing it would yield a bare "MLU-" that reads as a valid id and is the same + for every card, so the usage join would write one card's metrics onto all of them. + """ + lib = install_fake_library(monkeypatch) + + def _empty_uuid(uuid_info, _device): + uuid_info.uuid = "" + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetUUID"] = _empty_uuid + + with pytest.raises(pycndev.CNDevError) as raised: + pycndev.cndevGetUUID(DEVICE) + + assert raised.value == pycndev.CNDEV_ERROR_NOT_FOUND + + +def test_get_pcie_bus_id_formats_the_bdf(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetPCIeBusId(DEVICE) == FAKE_PCIE_BUS_ID + + +def test_get_card_name_prefers_the_driver_string(monkeypatch): + lib = install_fake_library(monkeypatch) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == FAKE_CARD_NAME + # The cheapest call answered, so the enum is never consulted. + assert "cndevGetCardName" not in lib.symbols + + +def test_get_card_name_falls_back_to_the_name_enum(monkeypatch): + lib = install_fake_library(monkeypatch, missing=("cndevGetCardNameStringByDevId",)) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU590" + assert "cndevGetCardNameString" in lib.symbols + + +def test_get_card_name_falls_back_to_the_name_table(monkeypatch): + """ + A driver exporting neither name string still yields a card name. + """ + lib = install_fake_library( + monkeypatch, + missing=("cndevGetCardNameStringByDevId", "cndevGetCardNameString"), + ) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU590" + assert "cndevGetCardName" in lib.symbols + + +def test_get_card_name_falls_back_to_the_family(monkeypatch): + lib = install_fake_library( + monkeypatch, + missing=("cndevGetCardNameStringByDevId", "cndevGetCardNameString"), + ) + + def unknown_card_name(card_name, _device): + card_name.id = 0x7FFF + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetCardName"] = unknown_card_name + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU" + + +def test_get_memory_usage_reports_the_physical_totals(monkeypatch): + install_fake_library(monkeypatch) + + memory_info = pycndev.cndevGetMemoryUsageV2(DEVICE) + + assert memory_info.physicalMemoryTotal == FAKE_MEMORY_TOTAL + assert memory_info.physicalMemoryUsed == FAKE_MEMORY_USED + + +def test_get_card_health_state_v2_falls_back_to_v1(monkeypatch): + """ + A driver without the V2 call still reports health, through the V1 struct. + """ + lib = install_fake_library(monkeypatch, missing=("cndevGetCardHealthStateV2",)) + + health_state = pycndev.cndevGetCardHealthStateV2(DEVICE) + + assert "cndevGetCardHealthState" in lib.symbols + assert health_state.health == 1 + assert health_state.deviceState == pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + assert health_state.driverState == pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + # V1 knows nothing about incidents, so the report stays empty rather than stale. + assert health_state.incident_count == 0 + # And the overall verdict is derived rather than left at zero, which is + # CNDEV_HEALTH_RESULT_PASS and would read as a pass on every old driver. + assert health_state.overallHealth == pycndev.CNDEV_HEALTH_RESULT_PASS + + +def test_get_card_health_state_v2_derives_a_failing_v1_verdict(monkeypatch): + lib = install_fake_library(monkeypatch, missing=("cndevGetCardHealthStateV2",)) + + def _in_problem(health_state, _device): + health_state.health = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetCardHealthState"] = _in_problem + + health_state = pycndev.cndevGetCardHealthStateV2(DEVICE) + + assert health_state.overallHealth == pycndev.CNDEV_HEALTH_RESULT_FAIL + + +def test_get_version_info_reports_the_driver_version(monkeypatch): + install_fake_library(monkeypatch) + + version_info = pycndev.cndevGetVersionInfo(DEVICE) + + assert ( + version_info.driverMajorVersion, + version_info.driverMinorVersion, + version_info.driverBuildVersion, + ) == FAKE_DRIVER_VERSION + + +def test_usage_wrappers_report_their_metrics(monkeypatch): + install_fake_library(monkeypatch) + + assert ( + pycndev.cndevGetDeviceUtilizationInfo(DEVICE).averageCoreUtilization + == FAKE_CORE_UTILIZATION + ) + assert pycndev.cndevGetTemperatureInfo(DEVICE).board == FAKE_BOARD_TEMPERATURE + assert pycndev.cndevGetDevicePowerInfo(DEVICE).usage == FAKE_POWER_USAGE + assert pycndev.cndevGetDevicePowerInfo(DEVICE).cap == FAKE_POWER_CAP + + +def test_get_numa_node_id_reports_the_node(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetNUMANodeIdByDevId(DEVICE).nodeId == FAKE_NUMA_NODE + + +def test_get_ecc_info_reports_the_counters(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetECCInfo(DEVICE).correctedError == FAKE_CORRECTED_ERRORS + + +def test_init_is_idempotent(monkeypatch): + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + pycndev.cndevInit() + pycndev.cndevInit() + + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_init_replays_a_cached_failure(monkeypatch): + """ + A failed init must keep failing, with a fresh exception each time. + + Re-raising the cached object appends a traceback frame per raise, and those frames + retain their callers' locals. See gpustack/gpustack#5342. + """ + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + lib.handlers["cndevInit"] = lambda *_: pycndev.CNDEV_ERROR_NO_DRIVER + + with pytest.raises(pycndev.CNDevError) as first: + pycndev.cndevInit() + with pytest.raises(pycndev.CNDevError) as second: + pycndev.cndevInit() + + assert first.value == pycndev.CNDEV_ERROR_NO_DRIVER + assert second.value == pycndev.CNDEV_ERROR_NO_DRIVER + assert first.value is not second.value + # The driver was asked once; the second call replayed the cached failure. + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_init_reaches_the_driver_once_when_two_callers_race(monkeypatch): + """ + Two concurrent first callers must not both reach the driver. + + The loser is answered with an already-initialized error, and caching that as the + library's permanent state disables Cambricon detection until the process restarts. + The operator's binding uses sync.Once here. + """ + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + entered = threading.Event() + entered_again = threading.Event() + proceed = threading.Event() + entries: list[int] = [] + + def _blocking_init(*_args): + entries.append(1) + (entered if len(entries) == 1 else entered_again).set() + # Stay inside the driver call until the second caller has had its chance. + proceed.wait(timeout=5) + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevInit"] = _blocking_init + + failures: list[BaseException] = [] + + def _init(): + try: + pycndev.cndevInit() + except BaseException as e: + failures.append(e) + + first = threading.Thread(target=_init) + first.start() + assert entered.wait(timeout=5) + + second = threading.Thread(target=_init) + second.start() + # The second caller has to wait for the lock the first one holds, so it must not + # reach the driver while the first call is still open. + assert not entered_again.wait(timeout=0.5) + + proceed.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not failures + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_release_reaches_the_driver(monkeypatch): + lib = install_fake_library(monkeypatch) + + pycndev.cndevRelease() + + assert [name for name, _ in lib.calls] == ["cndevRelease"] + + +def test_release_without_init_is_a_no_op(monkeypatch): + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + pycndev.cndevRelease() + + assert lib.calls == [] + + +def test_release_tolerates_a_driver_without_the_symbol(monkeypatch): + install_fake_library(monkeypatch, missing=("cndevRelease",)) + + pycndev.cndevRelease() From 167c060279ef0f7dcc6963723d948d88886b2934 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:25 +0800 Subject: [PATCH 05/12] fix(detector): name amd devices as the operator does, and split their usage - resolve the name from the local PCI ID database first, then the HSA product name, then libdrm's marketing name, then the ASIC name, which is the operator's full precedence, so a board reads as the board - bind `amdgpu_get_marketing_name` in `pyamdgpu`, which bound no such call and so left that step unreachable; declare its return type, since it answers with a pointer into libdrm's own table, and report an unnamed board as empty rather than as an error -- libdrm answers NULL for a device id its table does not carry, and the caller simply falls through to its next source - open the device only when pci.ids and HSA both found nothing, which is the rare path, so the common one costs no extra open. Verified against real libdrm on the AMD host: both cards answer "AMD Radeon RX 7800 XT". - move utilization, temperature, used memory and used power into `detect_usage`, leaving the power limit in the inventory query - drop the `vgpu` appendix key and the SR-IOV physical-function comparison - cover the name precedence, the split and the unchanged CDI device paths Signed-off-by: thxCode --- gpustack_runtime/detector/amd.py | 256 ++++++-- .../detector/pyamdgpu/__init__.py | 13 + tests/gpustack_runtime/detector/test_amd.py | 549 ++++++++++++++++++ 3 files changed, 770 insertions(+), 48 deletions(-) diff --git a/gpustack_runtime/detector/amd.py b/gpustack_runtime/detector/amd.py index f905e45..d8e1f8b 100644 --- a/gpustack_runtime/detector/amd.py +++ b/gpustack_runtime/detector/amd.py @@ -15,6 +15,7 @@ Devices, ManufacturerEnum, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -22,14 +23,26 @@ compare_pci_devices, get_brief_version, get_numa_node_by_bdf, + get_pci_device_name, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) logger = logging.getLogger(__name__) +_PCI_DEVICES_PATH = Path("/sys/bus/pci/devices") +""" +Location where sysfs exposes the PCI devices, +which a device's PCI IDs are read from. +""" + +_PCI_ID_FILES = ("vendor", "device", "subsystem_vendor", "subsystem_device") +""" +The sysfs files holding a PCI device's IDs, +in the order the PCI ID database is queried with. +""" + class AMDDetector(Detector): """ @@ -76,9 +89,10 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.AMD) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect AMD GPUs using pyamdsmi, pyamdgpu and pyrocmsmi. + Detect AMD GPUs' inventory using pyamdsmi, pyamdgpu and pyrocmsmi, + without usage metrics. Returns: A list of detected AMD GPU devices, @@ -113,11 +127,7 @@ def detect(self) -> Devices | None: dev_index = dev_idx dev_gpu_asic_info = pyamdsmi.amdsmi_get_gpu_asic_info(dev) - if dev_gpu_asic_info.get("asic_serial") != "N/A": - asic_serial = dev_gpu_asic_info.get("asic_serial") - dev_uuid = f"GPU-{(asic_serial[2:]).lower()}" - else: - dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + dev_uuid = _get_device_uuid(dev_gpu_asic_info, dev_idx) dev_bdf = pyamdsmi.amdsmi_get_gpu_device_bdf(dev) dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf) @@ -129,7 +139,23 @@ def detect(self) -> Devices | None: dev_gpu_driver_info = pyamdsmi.amdsmi_get_gpu_driver_info(dev) dev_driver_ver = dev_gpu_driver_info.get("driver_version") - dev_name = dev_hsa_agent.name + # The operator resolves the name from the local PCI ID database + # first: pci.ids knows the board -- the subsystem vendor's name + # for the card -- where the driver only knows the chip. The + # driver-reported names stay as fallbacks. + dev_name = _get_pci_device_name_by_bdf(dev_bdf) + if not dev_name: + dev_name = dev_hsa_agent.name + if not dev_name and dev_card_id is not None: + # The operator asks libdrm for the board's marketing name + # between the HSA and the ASIC name, so this is that step. + # Reached only when the two above found nothing, which is + # why the device is opened here rather than up front. + with ( + contextlib.suppress(pyamdgpu.AMDGPUError), + pyamdgpu.amdgpu_device(dev_card_id) as dev_gpudev, + ): + dev_name = pyamdgpu.amdgpu_get_marketing_name(dev_gpudev) if not dev_name: dev_name = dev_gpu_asic_info.get("market_name") @@ -158,31 +184,11 @@ def detect(self) -> Devices | None: if not dev_asic_family_id: dev_asic_family_id = dev_gpudev_info.family_id - dev_cores_util = None - dev_temp = None - try: - dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev) - dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0) - dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0) - except pyamdsmi.AmdSmiException: - with contextlib.suppress(pyrocmsmi.ROCMSMIError): - dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) - dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - dev_mem = 0 - dev_mem_used = 0 dev_mem_status = DeviceMemoryStatusEnum.HEALTHY try: dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) dev_mem = dev_gpu_vram_usage.get("vram_total") - dev_mem_used = dev_gpu_vram_usage.get("vram_used") if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( dev, @@ -194,9 +200,6 @@ def detect(self) -> Devices | None: dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), - ) if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyrocmsmi.ROCMSMIError): dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( @@ -205,24 +208,17 @@ def detect(self) -> Devices | None: if dev_ecc_count.uncorrectable_err > 0: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # The power limit is inventory, so it stays here, while the used + # power the same call carries belongs to the usage query. dev_power = None - dev_power_used = None try: dev_power_info = pyamdsmi.amdsmi_get_power_info(dev) dev_power = ( dev_power_info.get("power_limit", 0) // 1000000 ) # uW to W - dev_power_used = ( - dev_power_info.get("current_socket_power") - if dev_power_info.get("current_socket_power", "N/A") != "N/A" - else dev_power_info.get("average_socket_power", 0) - ) except pyamdsmi.AmdSmiException: with contextlib.suppress(pyrocmsmi.ROCMSMIError): dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx) - dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) - - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: @@ -231,7 +227,6 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _get_arch_family(dev_asic_family_id), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -255,14 +250,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -275,6 +265,126 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch AMD GPUs' usage using pyamdsmi and pyrocmsmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyamdsmi.amdsmi_init() + try: + pyrocmsmi.rsmi_init() + except Exception: + debug_log_exception(logger, "Failed to initialize ROCm SMI") + + devs = pyamdsmi.amdsmi_get_processor_handles() + for dev_idx, dev in enumerate(devs): + dev_gpu_asic_info = pyamdsmi.amdsmi_get_gpu_asic_info(dev) + dev_uuid = _get_device_uuid(dev_gpu_asic_info, dev_idx) + + dev_cores_util = None + dev_temp = None + try: + dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev) + dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0) + dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0) + except pyamdsmi.AmdSmiException: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) + dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem = 0 + dev_mem_used = 0 + # Health is reported by both queries, as the operator reports it + # from DetectAccelerator and MonitorAccelerator alike. + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + try: + dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) + dev_mem = dev_gpu_vram_usage.get("vram_total") + dev_mem_used = dev_gpu_vram_usage.get("vram_used") + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( + dev, + pyamdsmi.AmdSmiGpuBlock.UMC, + ) + if dev_ecc_count.get("uncorrectable_count", 0) > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + except pyamdsmi.AmdSmiException: + dev_mem = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), + ) + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( + dev_idx, + ) + if dev_ecc_count.uncorrectable_err > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_power_used = None + try: + dev_power_info = pyamdsmi.amdsmi_get_power_info(dev) + dev_power_used = ( + dev_power_info.get("current_socket_power") + if dev_power_info.get("current_socket_power", "N/A") != "N/A" + else dev_power_info.get("average_socket_power", 0) + ) + except pyamdsmi.AmdSmiException: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyamdsmi.AmdSmiException: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between AMD GPUs. @@ -289,7 +399,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -400,6 +510,56 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum: return ret +def _get_pci_device_name_by_bdf(dev_bdf: str) -> str: + """ + Get the name of a device from the local PCI ID database. + + Mirrors the operator, which prefers this name over every driver-reported + one: pci.ids knows the board -- the subsystem vendor's name for the card -- + where the driver only knows the chip. + + Args: + dev_bdf: + The device bdf. + + Returns: + The name of the device, + or an empty string if the database or the device is unknown. + + """ + if not dev_bdf: + return "" + + dev_pci_ids = [] + for id_file in _PCI_ID_FILES: + dev_pci_id = "" + with contextlib.suppress(OSError): + dev_pci_id = (_PCI_DEVICES_PATH / dev_bdf / id_file).read_text().strip() + dev_pci_ids.append(dev_pci_id) + + return get_pci_device_name(*dev_pci_ids) + + +def _get_device_uuid(dev_gpu_asic_info: dict, dev_idx: int) -> str: + """ + Get the UUID of a device. + + Args: + dev_gpu_asic_info: + The ASIC information of the device. + dev_idx: + The index of the device. + + Returns: + The UUID of the device. + + """ + if dev_gpu_asic_info.get("asic_serial") != "N/A": + asic_serial = dev_gpu_asic_info.get("asic_serial") + return f"GPU-{(asic_serial[2:]).lower()}" + return f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + + def _get_arch_family(dev_family_id: int | None) -> str: """ Get the architecture family name from the device family ID. diff --git a/gpustack_runtime/detector/pyamdgpu/__init__.py b/gpustack_runtime/detector/pyamdgpu/__init__.py index a1ddba6..afc2f80 100644 --- a/gpustack_runtime/detector/pyamdgpu/__init__.py +++ b/gpustack_runtime/detector/pyamdgpu/__init__.py @@ -284,3 +284,16 @@ def amdgpu_query_gpu_info(device): ret = fn(device, byref(c_info)) _amdgpuCheckReturn(ret) return c_info + + +def amdgpu_get_marketing_name(device): + fn = _amdgpuGetFunctionPointer("amdgpu_get_marketing_name") + # This one answers with a pointer into libdrm's own table rather than + # filling a buffer, so the return type has to be declared before the call + # or ctypes truncates the pointer to an int. + fn.restype = c_char_p + c_name = fn(device) + # The table carries no entry for every device id, and libdrm answers NULL + # rather than failing, so an unnamed board is empty here and the caller + # falls through to its next name source. + return c_name.decode() if c_name else "" diff --git a/tests/gpustack_runtime/detector/test_amd.py b/tests/gpustack_runtime/detector/test_amd.py index 402f539..918b961 100644 --- a/tests/gpustack_runtime/detector/test_amd.py +++ b/tests/gpustack_runtime/detector/test_amd.py @@ -1,5 +1,21 @@ +from __future__ import annotations + +import contextlib +from types import SimpleNamespace + import pytest +from gpustack_runtime.deployer.cdi import amd as cdi_amd +from gpustack_runtime.deployer.cdi.__types__ import ConfigDeviceNode +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + amd, + pyamdgpu, + pyhsa, +) +from gpustack_runtime.detector.__utils__ import _load_pci_device_names from gpustack_runtime.detector.amd import AMDDetector @@ -21,3 +37,536 @@ def test_get_topology(): det = AMDDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# Fake bindings: no AMD driver exists in this suite, so every behaviour below # +# is proved against stand-ins for pyamdsmi / pyrocmsmi / pyhsa, each recording # +# the calls it serves -- "detect_info asks for no usage" is a statement about # +# the calls made, not about the values returned. # +# --------------------------------------------------------------------------- # + + +class _FakeAmdSmiError(Exception): + """ + The fake AMD SMI's own error type. + + The detector catches whatever the binding module it is handed exposes, so + the fake brings its own error rather than constructing the real amdsmi + package's, which takes a status code and is absent here anyway. + """ + + +class _FakeRocmSmiError(Exception): + """ + The fake ROCm SMI's own error type. + """ + + +class _FakeAmdSmi: + """ + A pyamdsmi stand-in whose processor handles are the fixture cards + themselves, so every per-device answer is read out of the card. + """ + + AmdSmiException = _FakeAmdSmiError + + class AmdSmiGpuBlock: + UMC = 1 + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def amdsmi_init(self, *_args): + self.calls.append("amdsmi_init") + + def amdsmi_get_rocm_version(self) -> str: + return "6.4.1-123" + + def amdsmi_get_processor_handles(self) -> list[dict]: + return self.cards + + def amdsmi_get_gpu_asic_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_asic_info") + return dev["asic_info"] + + def amdsmi_get_gpu_device_bdf(self, dev: dict) -> str: + return dev["bdf"] + + def amdsmi_get_gpu_driver_info(self, dev: dict) -> dict: + return dev["driver_info"] + + def amdsmi_get_gpu_vram_usage(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_vram_usage") + return dev["vram"] + + def amdsmi_get_gpu_ecc_count(self, dev: dict, _block: int) -> dict: + self.calls.append("amdsmi_get_gpu_ecc_count") + return dev["ecc"] + + def amdsmi_get_power_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_power_info") + return dev["power"] + + def amdsmi_get_gpu_metrics_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_metrics_info") + if dev["metrics"] is None: + msg = "GPU metrics are not supported" + raise _FakeAmdSmiError(msg) + return dev["metrics"] + + def amdsmi_topo_get_numa_node_number(self, dev: dict) -> int: + return dev["numa"] + + def amdsmi_get_xgmi_info(self, dev: dict) -> dict: + return dev["xgmi"] + + +class _FakeRocmSmi: + """ + A pyrocmsmi stand-in serving the fallbacks the AMD path keeps for a driver + whose AMD SMI answers nothing. + """ + + ROCMSMIError = _FakeRocmSmiError + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def rsmi_init(self, *_args): + self.calls.append("rsmi_init") + + def rsmi_dev_busy_percent_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_busy_percent_get") + return self.cards[dev_idx]["busy_percent"] + + def rsmi_dev_temp_metric_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_temp_metric_get") + return self.cards[dev_idx]["temperature"] + + def rsmi_dev_power_cap_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_cap_get") + return self.cards[dev_idx]["power_cap"] + + def rsmi_dev_power_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_get") + return self.cards[dev_idx]["power_used"] + + +class _FakeHSA: + """ + A pyhsa stand-in returning the fixture agents. + """ + + Agent = pyhsa.Agent + + def __init__(self, agents: list): + self.agents = agents + + def get_agents(self) -> list: + return self.agents + + +# An extract of the real pci.ids: the board's name lives on the subsystem line, +# which is what the operator's GetName prefers. +_PCI_IDS = """\ +1002 Advanced Micro Devices, Inc. [AMD/ATI] +\t74a1 Aqua Vanjaram +\t\t1002 0e3b Instinct MI300X OAM +""" + +_PCI_IDS_NAME = "Instinct MI300X OAM" +_HSA_NAME = "AMD Instinct MI300X" +_ASIC_MARKET_NAME = "Aqua Vanjaram" +_MARKETING_NAME = "AMD Instinct MI300X OAM" + + +class _FakeAmdGpu: + """ + A libdrm stand-in exposing only what the name precedence reaches for. + + The real call answers with a pointer into libdrm's own table, and NULL for + a device id the table does not carry, which the binding reports as "". + """ + + def __init__(self, marketing_name: str): + self._marketing_name = marketing_name + + def __getattr__(self, name): + # Everything not faked here -- the error type and the family constants + # -- comes from the real binding, so the fake cannot drift from it. + return getattr(pyamdgpu, name) + + @contextlib.contextmanager + def amdgpu_device(self, card): + yield SimpleNamespace(card=card) + + def amdgpu_get_marketing_name(self, _device) -> str: + return self._marketing_name + + def amdgpu_query_gpu_info(self, _device): + return SimpleNamespace(cu_active_number=0, family_id=0) + + +_MEMORY_TOTAL = 196592 + + +def _card( + bdf: str, + serial: str, + *, + gfx_activity: int = 37, + hotspot: int = 58, + vram_used: int = 1024, + socket_power: int = 142, + no_metrics: bool = False, +) -> dict: + """ + One fake AMD card: the dict is the processor handle the fake AMD SMI hands + out, and the PCI IDs the fixture sysfs tree exposes for its BDF. + """ + return { + "bdf": bdf, + "pci_ids": { + "vendor": "0x1002", + "device": "0x74a1", + "subsystem_vendor": "0x1002", + "subsystem_device": "0x0e3b", + }, + "asic_info": { + "market_name": _ASIC_MARKET_NAME, + "asic_serial": serial, + "target_graphics_version": "gfx942", + }, + "driver_info": {"driver_version": "6.12.12"}, + "vram": {"vram_total": _MEMORY_TOTAL, "vram_used": vram_used}, + "ecc": {"uncorrectable_count": 0}, + "power": { + "power_limit": 750000000, + "current_socket_power": socket_power, + "average_socket_power": 138, + }, + "metrics": None + if no_metrics + else { + "average_gfx_activity": gfx_activity, + "temperature_hotspot": hotspot, + }, + "busy_percent": 23, + "temperature": 47, + "power_cap": 700, + "power_used": 131, + "numa": 0, + "xgmi": {"xgmi_lanes": 16, "xgmi_hive_id": "0x2b", "xgmi_node_id": 3}, + } + + +def _agent(bdf: str) -> pyhsa.Agent: + return pyhsa.Agent( + device_type=1, + device_id="0x74a1", + bdf=bdf, + uuid="", + name=_HSA_NAME, + compute_capability="gfx942", + compute_units=304, + ) + + +@pytest.fixture +def amd_bindings(monkeypatch, tmp_path): + """ + Drive the AMD detector off the fake bindings, a fixture sysfs PCI tree and + a fixture pci.ids database, returning the shared call log. + """ + + def _setup( + cards: list[dict], + agents: list | None = None, + pci_ids: str | None = _PCI_IDS, + ) -> list[str]: + calls: list[str] = [] + + monkeypatch.setattr(AMDDetector, "is_supported", staticmethod(lambda: True)) + monkeypatch.setattr(amd, "pyamdsmi", _FakeAmdSmi(calls, cards)) + monkeypatch.setattr(amd, "pyrocmsmi", _FakeRocmSmi(calls, cards)) + monkeypatch.setattr(amd, "pyhsa", _FakeHSA(list(agents or []))) + + pci_devices_path = tmp_path / "pci_devices" + for card in cards: + card_path = pci_devices_path / card["bdf"] + card_path.mkdir(parents=True) + for name, value in card["pci_ids"].items(): + (card_path / name).write_text(f"{value}\n") + monkeypatch.setattr(amd, "_PCI_DEVICES_PATH", pci_devices_path) + + pci_ids_paths: tuple[str, ...] = () + if pci_ids is not None: + pci_ids_path = tmp_path / "pci.ids" + pci_ids_path.write_text(pci_ids, encoding="utf-8") + pci_ids_paths = (str(pci_ids_path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + pci_ids_paths, + ) + _load_pci_device_names.cache_clear() + + return calls + + yield _setup + + _load_pci_device_names.cache_clear() + + +# --------------------------------------------------------------------------- # +# detect_info: the device name's precedence, the inventory, and the usage # +# calls it must not make. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_pci_ids_name(amd_bindings): + # The operator resolves the board's name from pci.ids before asking the + # driver, which only knows the chip. + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_PCI_IDS_NAME] + + +def test_detect_info_falls_back_to_the_hsa_product_name(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + pci_ids=None, + ) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_HSA_NAME] + + +def test_detect_info_falls_back_to_the_asic_market_name(amd_bindings): + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_ASIC_MARKET_NAME] + + +def test_detect_info_asks_libdrm_for_the_marketing_name(amd_bindings, monkeypatch): + # Between the HSA name and the ASIC name the operator asks libdrm for the + # board's marketing name. With pci.ids and HSA both silent, that answer is + # the one reported -- the ASIC market name is only the step after it. + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + monkeypatch.setattr(amd, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr(amd, "pyamdgpu", _FakeAmdGpu(marketing_name=_MARKETING_NAME)) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_MARKETING_NAME] + + +def test_detect_info_falls_through_an_unnamed_board(amd_bindings, monkeypatch): + # libdrm's table carries no entry for every device id and answers NULL + # rather than failing, which the binding reports as an empty name. + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + monkeypatch.setattr(amd, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr(amd, "pyamdgpu", _FakeAmdGpu(marketing_name="")) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_ASIC_MARKET_NAME] + + +def test_detect_info_reports_the_inventory(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00A1B2C3D4E5F600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + dev = devices[0] + assert dev.manufacturer is ManufacturerEnum.AMD + assert dev.index == 0 + # The ASIC serial, lowercased and stripped of its "0x". + assert dev.uuid == "GPU-00a1b2c3d4e5f600" + assert dev.driver_version == "6.12.12" + assert dev.runtime_version == "6.4" + assert dev.runtime_version_original == "6.4.1-123" + assert dev.compute_capability == "gfx942" + assert dev.cores == 304 + assert dev.memory == _MEMORY_TOTAL + assert dev.memory_status is DeviceMemoryStatusEnum.HEALTHY + # The power limit is inventory, not usage. + assert dev.power == 750 + assert dev.appendix["bdf"] == "0000:05:00.0" + assert dev.appendix["numa"] == "0" + assert dev.appendix["xgmi_lanes"] == 16 + + +def test_detect_reports_no_vgpu(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + det = AMDDetector() + + # There is no virtual/PF/VF classification any more, in either query. + assert "vgpu" not in det.detect_info()[0].appendix + assert "vgpu" not in det.detect()[0].appendix + + +def test_detect_info_issues_no_usage_call(amd_bindings): + calls = amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + assert "amdsmi_get_gpu_metrics_info" not in calls + assert "rsmi_dev_busy_percent_get" not in calls + assert "rsmi_dev_temp_metric_get" not in calls + assert "rsmi_dev_power_get" not in calls + # The power query stays, because the limit it also carries is inventory. + assert "amdsmi_get_power_info" in calls + + dev = devices[0] + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +# --------------------------------------------------------------------------- # +# detect_usage. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(amd_bindings): + amd_bindings( + [ + _card("0000:05:00.0", "0x00a1b2c3d4e5f600"), + _card( + "0000:06:00.0", + "0x00a1b2c3d4e5f601", + gfx_activity=91, + hotspot=72, + vram_used=8192, + socket_power=311, + ), + ], + agents=[_agent("0000:05:00.0"), _agent("0000:06:00.0")], + ) + det = AMDDetector() + devices = det.detect_info() + + # Reversed, to prove the merge joins by UUID rather than by position. + det.detect_usage(list(reversed(devices))) + + assert [dev.cores_utilization for dev in devices] == [37, 91] + assert [dev.temperature for dev in devices] == [58, 72] + assert [dev.memory_used for dev in devices] == [1024, 8192] + assert [dev.power_used for dev in devices] == [142, 311] + assert [dev.memory_utilization for dev in devices] == [0.52, 4.17] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + DeviceMemoryStatusEnum.HEALTHY, + ] + # The information fields survive the merge. + assert [dev.memory for dev in devices] == [_MEMORY_TOTAL, _MEMORY_TOTAL] + assert [dev.power for dev in devices] == [750, 750] + + +def test_detect_usage_falls_back_to_rocm_smi(amd_bindings): + calls = amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600", no_metrics=True)], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect() + + assert "rsmi_dev_busy_percent_get" in calls + assert "rsmi_dev_temp_metric_get" in calls + assert devices[0].cores_utilization == 23 + assert devices[0].temperature == 47 + + +def test_detect_usage_detects_the_information_first(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_usage() + + assert devices[0].name == _PCI_IDS_NAME + assert devices[0].cores_utilization == 37 + assert devices[0].power_used == 142 + + +def test_detect_composes_the_information_and_the_usage(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect() + + assert devices[0].memory == _MEMORY_TOTAL + assert devices[0].memory_used == 1024 + assert devices[0].power == 750 + assert devices[0].power_used == 142 + + +# --------------------------------------------------------------------------- # +# The CDI generator, which numbers its device nodes from the appendix. # +# --------------------------------------------------------------------------- # + + +def _fake_device_node(path: str, **_kwargs) -> ConfigDeviceNode: + """ + Stand in for the host's device nodes: neither /dev/kfd nor /dev/dri exists + in this suite, so the real lookup would drop every path. + """ + return ConfigDeviceNode(path=path, type_="c") + + +def test_cdi_spec_numbers_the_device_nodes_from_the_appendix(monkeypatch): + monkeypatch.setattr(cdi_amd, "device_to_cdi_device_node", _fake_device_node) + + config = cdi_amd.AMDGenerator().generate( + [ + Device( + manufacturer=ManufacturerEnum.AMD, + index=0, + name=_PCI_IDS_NAME, + uuid="GPU-a1b2c3d4e5f600", + appendix={"card_id": 5, "renderd_id": 133}, + ), + ], + ) + + # The DRM numbering comes from the appendix, never from Device.index -- + # here the card is enumerated at 0 and its nodes are card5 / renderD133. + device_nodes = config["devices"][0]["containerEdits"]["deviceNodes"] + assert [node["path"] for node in device_nodes] == [ + "/dev/dri/card5", + "/dev/dri/renderD133", + ] + assert [dev["name"] for dev in config["devices"]] == [ + "0", + "GPU-a1b2c3d4e5f600", + "all", + ] + assert [node["path"] for node in config["containerEdits"]["deviceNodes"]] == [ + "/dev/kfd", + ] From a18d5bf0056e757fec0051484f539379ac3d9767 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:31 +0800 Subject: [PATCH 06/12] fix(detector): name hygon devices as the operator does, and split their usage - adopt the same name precedence AMD now has -- the local PCI ID database, then the HSA product name, then libdrm's marketing name, then the driver-reported name -- since a Hygon DCU runs on the same ROCm binding stack - move utilization, temperature, used memory and used power into `detect_usage`, leaving the power limit in the inventory query - drop the `vgpu` appendix key and the SR-IOV physical-function comparison - cover the name precedence, the split and the unchanged CDI device paths Signed-off-by: thxCode --- gpustack_runtime/detector/hygon.py | 135 ++++- tests/gpustack_runtime/detector/test_hygon.py | 498 ++++++++++++++++++ 2 files changed, 605 insertions(+), 28 deletions(-) diff --git a/gpustack_runtime/detector/hygon.py b/gpustack_runtime/detector/hygon.py index 2bc93d3..fd704b5 100644 --- a/gpustack_runtime/detector/hygon.py +++ b/gpustack_runtime/detector/hygon.py @@ -15,6 +15,7 @@ Devices, ManufacturerEnum, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -23,11 +24,10 @@ get_brief_version, get_numa_node_by_bdf, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) -from .amd import _get_arch_family +from .amd import _get_arch_family, _get_pci_device_name_by_bdf logger = logging.getLogger(__name__) @@ -77,9 +77,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.HYGON) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Hygon GPUs using pyrocmsmi. + Detect Hygon GPUs' inventory using pyrocmsmi, without usage metrics. Returns: A list of detected Hygon GPU devices, @@ -128,7 +128,23 @@ def detect(self) -> Devices | None: hsa_agents.get(dev_bdf) or hsa_agents.get(dev_uuid) or pyhsa.Agent() ) - dev_name = dev_hsa_agent.name + # The operator resolves the name from the local PCI ID database + # first: pci.ids knows the board -- the subsystem vendor's name + # for the card -- where the driver only knows the chip. The + # driver-reported names stay as fallbacks. + dev_name = _get_pci_device_name_by_bdf(dev_bdf) + if not dev_name: + dev_name = dev_hsa_agent.name + if not dev_name and dev_card_id is not None: + # The operator asks libdrm for the board's marketing name + # between the HSA and the driver name, so this is that step. + # Reached only when the two above found nothing, which is + # why the device is opened here rather than up front. + with ( + contextlib.suppress(pyamdgpu.AMDGPUError), + pyamdgpu.amdgpu_device(dev_card_id) as dev_gpudev, + ): + dev_name = pyamdgpu.amdgpu_get_marketing_name(dev_gpudev) if not dev_name: dev_name = pyrocmsmi.rsmi_dev_name_get(dev_idx) @@ -152,22 +168,9 @@ def detect(self) -> Devices | None: if not dev_asic_family_id: dev_asic_family_id = dev_gpudev_info.family_id - dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) - dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), - ) dev_mem_status = DeviceMemoryStatusEnum.HEALTHY if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyrocmsmi.ROCMSMIError): @@ -177,10 +180,9 @@ def detect(self) -> Devices | None: if dev_ecc_count.uncorrectable_err > 0: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # The power limit is inventory, while the used power belongs to + # the usage query. dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx) - dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) - - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: @@ -191,7 +193,6 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _get_arch_family(dev_asic_family_id), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -212,14 +213,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -232,6 +228,89 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Hygon GPUs' usage using pyrocmsmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyrocmsmi.rsmi_init() + + devs_count = pyrocmsmi.rsmi_num_monitor_devices() + for dev_idx in range(devs_count): + dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + + dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) + dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), + ) + # Health is reported by both queries, as the operator reports it + # from DetectAccelerator and MonitorAccelerator alike. + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( + dev_idx, + ) + if dev_ecc_count.uncorrectable_err > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyrocmsmi.ROCMSMIError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between Hygon GPUs. @@ -246,7 +325,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None diff --git a/tests/gpustack_runtime/detector/test_hygon.py b/tests/gpustack_runtime/detector/test_hygon.py index d304ecc..db37bc0 100644 --- a/tests/gpustack_runtime/detector/test_hygon.py +++ b/tests/gpustack_runtime/detector/test_hygon.py @@ -1,5 +1,23 @@ +from __future__ import annotations + +import contextlib +from types import SimpleNamespace + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import hygon as cdi_hygon +from gpustack_runtime.deployer.cdi.__types__ import ConfigDeviceNode +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + amd, + hygon, + pyamdgpu, + pyhsa, +) +from gpustack_runtime.detector.__utils__ import _load_pci_device_names from gpustack_runtime.detector.hygon import HygonDetector @@ -21,3 +39,483 @@ def test_get_topology(): det = HygonDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# Fake bindings: no Hygon driver exists in this suite, so every behaviour # +# below is proved against stand-ins for pyrocmsmi / pyhsa, recording the calls # +# they serve -- "detect_info asks for no usage" is a statement about the calls # +# made, not about the values returned. # +# --------------------------------------------------------------------------- # + + +class _FakeRocmSmiError(Exception): + """ + The fake ROCm SMI's own error type. + """ + + +class _EccCount: + """ + An rsmi_error_count_t stand-in. + """ + + def __init__(self, uncorrectable_err: int = 0): + self.uncorrectable_err = uncorrectable_err + + +class _FakeRocmSmi: + """ + A pyrocmsmi stand-in answering per device index out of the fixture cards. + """ + + ROCMSMIError = _FakeRocmSmiError + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def rsmi_init(self, *_args): + self.calls.append("rsmi_init") + + def rsmi_get_rocm_version(self) -> str: + return "25.04.1" + + def rsmi_num_monitor_devices(self) -> int: + return len(self.cards) + + def rsmi_dev_unique_id_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["unique_id"] + + def rsmi_dev_pci_id_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["bdf"] + + def rsmi_dev_name_get(self, dev_idx: int) -> str: + self.calls.append("rsmi_dev_name_get") + return self.cards[dev_idx]["name"] + + def rsmi_dev_target_graphics_version_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["target_graphics_version"] + + def rsmi_dev_memory_total_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_memory_total_get") + return self.cards[dev_idx]["memory_total"] + + def rsmi_dev_memory_usage_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_memory_usage_get") + return self.cards[dev_idx]["memory_used"] + + def rsmi_dev_ecc_count_get(self, dev_idx: int) -> _EccCount: + self.calls.append("rsmi_dev_ecc_count_get") + return _EccCount(self.cards[dev_idx]["uncorrectable_err"]) + + def rsmi_dev_busy_percent_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_busy_percent_get") + return self.cards[dev_idx]["busy_percent"] + + def rsmi_dev_temp_metric_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_temp_metric_get") + return self.cards[dev_idx]["temperature"] + + def rsmi_dev_power_cap_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_cap_get") + return self.cards[dev_idx]["power_cap"] + + def rsmi_dev_power_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_get") + return self.cards[dev_idx]["power_used"] + + def rsmi_topo_get_numa_node_number(self, dev_idx: int) -> int: + return self.cards[dev_idx]["numa"] + + +class _FakeAmdGpu: + """ + A libdrm stand-in exposing only what the name precedence reaches for. + + The real call answers with a pointer into libdrm's own table, and NULL for + a device id the table does not carry, which the binding reports as "". + """ + + def __init__(self, marketing_name: str): + self._marketing_name = marketing_name + + def __getattr__(self, name): + # Everything not faked here comes from the real binding, so the fake + # cannot drift from it. + return getattr(pyamdgpu, name) + + @contextlib.contextmanager + def amdgpu_device(self, card): + yield SimpleNamespace(card=card) + + def amdgpu_get_marketing_name(self, _device) -> str: + return self._marketing_name + + def amdgpu_query_gpu_info(self, _device): + return SimpleNamespace(cu_active_number=0, family_id=0) + + +class _FakeHSA: + """ + A pyhsa stand-in returning the fixture agents. + """ + + Agent = pyhsa.Agent + + def __init__(self, agents: list): + self.agents = agents + + def get_agents(self) -> list: + return self.agents + + +# An extract of the real pci.ids: the board's name lives on the subsystem line, +# which is what the operator's GetName prefers. +_PCI_IDS = """\ +1d94 Chengdu Haiguang IC Design Co., Ltd. +\t6210 Kunpeng +\t\t1d94 6210 K100_AI +""" + +_PCI_IDS_NAME = "K100_AI" +_HSA_NAME = "Hygon K100 AI" +_MARKETING_NAME = "Hygon DCU K100 AI" +_RSMI_NAME = "Kunpeng" + +_MEMORY_TOTAL_BYTES = 68719476736 # 65536 MiB +_MEMORY_TOTAL = 65536 + + +def _card( + bdf: str, + unique_id: str, + *, + memory_used: int = 1073741824, # 1024 MiB + busy_percent: int = 44, + temperature: int = 51, + power_used: int = 217, + uncorrectable_err: int = 0, +) -> dict: + """ + One fake Hygon card: the answers the fake ROCm SMI serves for its index, + and the PCI IDs the fixture sysfs tree exposes for its BDF. + """ + return { + "bdf": bdf, + "pci_ids": { + "vendor": "0x1d94", + "device": "0x6210", + "subsystem_vendor": "0x1d94", + "subsystem_device": "0x6210", + }, + "unique_id": unique_id, + "name": _RSMI_NAME, + "target_graphics_version": "gfx936", + "memory_total": _MEMORY_TOTAL_BYTES, + "memory_used": memory_used, + "uncorrectable_err": uncorrectable_err, + "busy_percent": busy_percent, + "temperature": temperature, + "power_cap": 350, + "power_used": power_used, + "numa": 0, + } + + +def _agent(bdf: str) -> pyhsa.Agent: + return pyhsa.Agent( + device_type=1, + device_id="0x6210", + bdf=bdf, + uuid="", + name=_HSA_NAME, + compute_capability="gfx936", + compute_units=104, + ) + + +@pytest.fixture +def hygon_bindings(monkeypatch, tmp_path): + """ + Drive the Hygon detector off the fake bindings, a fixture sysfs PCI tree + and a fixture pci.ids database, returning the shared call log. + """ + + def _setup( + cards: list[dict], + agents: list | None = None, + pci_ids: str | None = _PCI_IDS, + ) -> list[str]: + calls: list[str] = [] + + monkeypatch.setattr(HygonDetector, "is_supported", staticmethod(lambda: True)) + monkeypatch.setattr(hygon, "pyrocmsmi", _FakeRocmSmi(calls, cards)) + monkeypatch.setattr(hygon, "pyhsa", _FakeHSA(list(agents or []))) + + pci_devices_path = tmp_path / "pci_devices" + for card in cards: + card_path = pci_devices_path / card["bdf"] + card_path.mkdir(parents=True) + for name, value in card["pci_ids"].items(): + (card_path / name).write_text(f"{value}\n") + # The pci.ids lookup is the AMD module's -- Hygon shares it, as it + # already shares the architecture family mapping. + monkeypatch.setattr(amd, "_PCI_DEVICES_PATH", pci_devices_path) + + pci_ids_paths: tuple[str, ...] = () + if pci_ids is not None: + pci_ids_path = tmp_path / "pci.ids" + pci_ids_path.write_text(pci_ids, encoding="utf-8") + pci_ids_paths = (str(pci_ids_path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + pci_ids_paths, + ) + _load_pci_device_names.cache_clear() + + return calls + + yield _setup + + _load_pci_device_names.cache_clear() + + +# --------------------------------------------------------------------------- # +# detect_info: the device name's precedence, the inventory, and the usage # +# calls it must not make. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_pci_ids_name(hygon_bindings): + # The operator resolves the board's name from pci.ids before asking the + # driver, which only knows the chip. + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_PCI_IDS_NAME] + + +def test_detect_info_falls_back_to_the_hsa_product_name(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + pci_ids=None, + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_HSA_NAME] + + +def test_detect_info_falls_back_to_the_rocm_smi_name(hygon_bindings): + hygon_bindings([_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], pci_ids=None) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_RSMI_NAME] + + +def test_detect_info_asks_libdrm_for_the_marketing_name(hygon_bindings, monkeypatch): + # The same libdrm step AMD gained: between the HSA name and the driver + # name, the operator asks for the board's marketing name. + hygon_bindings([_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], pci_ids=None) + monkeypatch.setattr(hygon, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr( + hygon, + "pyamdgpu", + _FakeAmdGpu(marketing_name=_MARKETING_NAME), + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_MARKETING_NAME] + + +def test_detect_info_reports_the_inventory(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + dev = devices[0] + assert dev.manufacturer is ManufacturerEnum.HYGON + assert dev.index == 0 + assert dev.uuid == "GPU-9f8e7d6c5b4a3921" + assert dev.runtime_version == "25.04" + assert dev.runtime_version_original == "25.04.1" + assert dev.compute_capability == "gfx936" + assert dev.cores == 104 + assert dev.memory == _MEMORY_TOTAL + assert dev.memory_status is DeviceMemoryStatusEnum.HEALTHY + # The power limit is inventory, not usage. + assert dev.power == 350 + assert dev.appendix["bdf"] == "0000:0b:00.0" + assert dev.appendix["numa"] == "0" + + +def test_detect_reports_no_vgpu(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + det = HygonDetector() + + # There is no virtual/PF/VF classification any more, in either query. + assert "vgpu" not in det.detect_info()[0].appendix + assert "vgpu" not in det.detect()[0].appendix + + +def test_detect_info_issues_no_usage_call(hygon_bindings): + calls = hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert "rsmi_dev_busy_percent_get" not in calls + assert "rsmi_dev_temp_metric_get" not in calls + assert "rsmi_dev_memory_usage_get" not in calls + assert "rsmi_dev_power_get" not in calls + # The power limit is inventory, so its own call stays. + assert "rsmi_dev_power_cap_get" in calls + + dev = devices[0] + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_reports_unhealthy_memory(hygon_bindings, monkeypatch): + # The ECC read is opt-in, as the health check is disabled by default. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921", uncorrectable_err=3)], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert devices[0].memory_status is DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# detect_usage. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(hygon_bindings): + hygon_bindings( + [ + _card("0000:0b:00.0", "0x9f8e7d6c5b4a3921"), + _card( + "0000:0c:00.0", + "0x9f8e7d6c5b4a3922", + memory_used=8589934592, # 8192 MiB + busy_percent=87, + temperature=63, + power_used=298, + ), + ], + agents=[_agent("0000:0b:00.0"), _agent("0000:0c:00.0")], + ) + det = HygonDetector() + devices = det.detect_info() + + # Reversed, to prove the merge joins by UUID rather than by position. + det.detect_usage(list(reversed(devices))) + + assert [dev.cores_utilization for dev in devices] == [44, 87] + assert [dev.temperature for dev in devices] == [51, 63] + assert [dev.memory_used for dev in devices] == [1024, 8192] + assert [dev.power_used for dev in devices] == [217, 298] + assert [dev.memory_utilization for dev in devices] == [1.56, 12.5] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + DeviceMemoryStatusEnum.HEALTHY, + ] + # The information fields survive the merge. + assert [dev.memory for dev in devices] == [_MEMORY_TOTAL, _MEMORY_TOTAL] + assert [dev.power for dev in devices] == [350, 350] + + +def test_detect_usage_detects_the_information_first(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_usage() + + assert devices[0].name == _PCI_IDS_NAME + assert devices[0].cores_utilization == 44 + assert devices[0].power_used == 217 + + +def test_detect_composes_the_information_and_the_usage(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect() + + assert devices[0].memory == _MEMORY_TOTAL + assert devices[0].memory_used == 1024 + assert devices[0].power == 350 + assert devices[0].power_used == 217 + + +# --------------------------------------------------------------------------- # +# The CDI generator, which numbers its device nodes from the appendix. # +# --------------------------------------------------------------------------- # + + +def _fake_device_node(path: str, **_kwargs) -> ConfigDeviceNode: + """ + Stand in for the host's device nodes: neither /dev/kfd nor /dev/dri exists + in this suite, so the real lookup would drop every path. + """ + return ConfigDeviceNode(path=path, type_="c") + + +def test_cdi_spec_numbers_the_device_nodes_from_the_appendix(monkeypatch): + monkeypatch.setattr(cdi_hygon, "device_to_cdi_device_node", _fake_device_node) + + config = cdi_hygon.HygonGenerator().generate( + [ + Device( + manufacturer=ManufacturerEnum.HYGON, + index=0, + name=_PCI_IDS_NAME, + uuid="GPU-9f8e7d6c5b4a3921", + appendix={"card_id": 3, "renderd_id": 131}, + ), + ], + ) + + # The DRM numbering comes from the appendix, never from Device.index -- + # here the card is enumerated at 0 and its nodes are card3 / renderD131. + device_nodes = config["devices"][0]["containerEdits"]["deviceNodes"] + assert [node["path"] for node in device_nodes] == [ + "/dev/dri/card3", + "/dev/dri/renderD131", + ] + assert [dev["name"] for dev in config["devices"]] == [ + "0", + "GPU-9f8e7d6c5b4a3921", + "all", + ] + assert [node["path"] for node in config["containerEdits"]["deviceNodes"]] == [ + "/dev/kfd", + "/dev/mkfd", + ] From e5dfd4267bacbd54896504c2797095db6c3b5f32 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:37 +0800 Subject: [PATCH 07/12] fix(detector): fall back to v1 memory on iluvatar, and split its usage query - read memory through the v2 structure with a v1 fallback, as the operator does, so a driver exposing only v1 still reports a device - move utilization, temperature and used power into `detect_usage` - drop the `vgpu` appendix key and the SR-IOV physical-function comparison - cover the fallback, the split and the unchanged `/dev/iluvatar{N}` path, which now reads the minor number from the appendix Signed-off-by: thxCode --- gpustack_runtime/deployer/cdi/iluvatar.py | 6 +- gpustack_runtime/detector/iluvatar.py | 189 ++++++++--- .../detector/test_iluvatar.py | 310 ++++++++++++++++++ 3 files changed, 461 insertions(+), 44 deletions(-) diff --git a/gpustack_runtime/deployer/cdi/iluvatar.py b/gpustack_runtime/deployer/cdi/iluvatar.py index e1d0648..5672be0 100644 --- a/gpustack_runtime/deployer/cdi/iluvatar.py +++ b/gpustack_runtime/deployer/cdi/iluvatar.py @@ -81,8 +81,12 @@ def generate( container_device_nodes = [] + # The device node is numbered by the driver's minor number, which + # Device.index no longer carries: it is the detector's enumeration + # index. Fall back to it for a device whose minor number could not + # be read. cdn = device_to_cdi_device_node( - path=f"/dev/iluvatar{dev.index}", + path=f"/dev/iluvatar{dev.appendix.get('minor_number', dev.index)}", ) if not cdn: continue diff --git a/gpustack_runtime/detector/iluvatar.py b/gpustack_runtime/detector/iluvatar.py index 211fcbe..5689519 100644 --- a/gpustack_runtime/detector/iluvatar.py +++ b/gpustack_runtime/detector/iluvatar.py @@ -15,6 +15,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -24,7 +25,6 @@ get_numa_node_by_bdf, get_numa_nodeset_size, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) @@ -77,9 +77,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.ILUVATAR) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Iluvatar GPUs using pyixml. + Detect Iluvatar GPUs' inventory using pyixml, without usage metrics. Returns: A list of detected Iluvatar GPU devices, @@ -122,9 +122,15 @@ def detect(self) -> Devices | None: dev = pyixml.nvmlDeviceGetHandleByIndex(dev_idx) dev_index = dev_idx - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - with contextlib.suppress(pyixml.NVMLError): - dev_index = pyixml.nvmlDeviceGetMinorNumber(dev) + + # Device.index is the enumeration index, while the driver's + # minor number is what /dev/iluvatar{N} is made of, so the + # latter goes to the appendix. Mirrors the operator, which + # keeps a sequential Index next to PhysicalIndexes, and omits + # the physical one when the driver cannot answer. + dev_minor_number = None + with contextlib.suppress(pyixml.NVMLError): + dev_minor_number = pyixml.nvmlDeviceGetMinorNumber(dev) dev_name = pyixml.nvmlDeviceGetName(dev) @@ -135,49 +141,25 @@ def detect(self) -> Devices | None: dev_cores = pyixml.nvmlDeviceGetNumGpuCores(dev) dev_mem = 0 - dev_mem_used = 0 dev_mem_status = DeviceMemoryStatusEnum.HEALTHY with contextlib.suppress(pyixml.NVMLError): - dev_mem_info = pyixml.nvmlDeviceGetMemoryInfo(dev) + # Prefer the v2 memory structure, falling back to v1 -- + # mirrors the operator's GetMemoryInfoV, which drops the + # device only when neither call succeeds. + dev_mem_info = _get_memory_info(dev) dev_mem = byte_to_mebibyte( # byte to MiB dev_mem_info.total, ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyixml.NVMLError): dev_health = pyixml.ixmlDeviceGetHealth(dev) if dev_health != pyixml.IXML_HEALTH_OK: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - dev_cores_util = None - with contextlib.suppress(pyixml.NVMLError): - dev_util_rates = pyixml.nvmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_temp = None - with contextlib.suppress(pyixml.NVMLError): - dev_temp = pyixml.nvmlDeviceGetTemperature( - dev, - pyixml.NVML_TEMPERATURE_GPU, - ) - dev_power = None - dev_power_used = None with contextlib.suppress(pyixml.NVMLError): dev_power = pyixml.nvmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pyixml.nvmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_cc = None with contextlib.suppress(pyixml.NVMLError): @@ -188,8 +170,6 @@ def detect(self) -> Devices | None: dev_pci_info = pyixml.nvmlDeviceGetPciInfo(dev) dev_bdf = str(dev_pci_info.busIdLegacy).lower() - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf - dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: with contextlib.suppress(pyixml.NVMLError): @@ -201,9 +181,10 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_numa: dev_appendix["numa"] = dev_numa @@ -218,14 +199,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -238,6 +214,106 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Iluvatar GPUs' usage using pyixml. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyixml.nvmlInit() + + dev_count = pyixml.nvmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pyixml.nvmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pyixml.nvmlDeviceGetUUID(dev) + + dev_mem = 0 + dev_mem_used = 0 + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + with contextlib.suppress(pyixml.NVMLError): + # Same v2-then-v1 fallback as detect_info: the operator's + # MonitorAccelerator re-reads the memory info rather than + # trusting a previous pass. + dev_mem_info = _get_memory_info(dev) + dev_mem = byte_to_mebibyte( # byte to MiB + dev_mem_info.total, + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + dev_mem_info.used, + ) + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyixml.NVMLError): + dev_health = pyixml.ixmlDeviceGetHealth(dev) + if dev_health != pyixml.IXML_HEALTH_OK: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_cores_util = None + with contextlib.suppress(pyixml.NVMLError): + dev_util_rates = pyixml.nvmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_temp = None + with contextlib.suppress(pyixml.NVMLError): + dev_temp = pyixml.nvmlDeviceGetTemperature( + dev, + pyixml.NVML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pyixml.NVMLError): + dev_power_used = ( + pyixml.nvmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyixml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -252,7 +328,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -302,3 +378,30 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: raise return ret + + +def _get_memory_info(dev): + """ + Read a device's memory info, preferring the v2 structure with a v1 + fallback. + + Mirrors the operator's GetMemoryInfoV, which tries + ``nvmlDeviceGetMemoryInfo_v2`` before falling back to the v1 call: the + runtime previously read the v1 accessor only, so a driver exposing just + v2 failed here where the operator succeeded. + + Args: + dev: + The device handle. + + Returns: + The memory info structure, from whichever accessor answered. + + Raises: + pyixml.NVMLError: If neither accessor succeeds. + + """ + try: + return pyixml.nvmlDeviceGetMemoryInfo(dev, version=pyixml.nvmlMemory_v2) + except pyixml.NVMLError: + return pyixml.nvmlDeviceGetMemoryInfo(dev) diff --git a/tests/gpustack_runtime/detector/test_iluvatar.py b/tests/gpustack_runtime/detector/test_iluvatar.py index f529912..999141a 100644 --- a/tests/gpustack_runtime/detector/test_iluvatar.py +++ b/tests/gpustack_runtime/detector/test_iluvatar.py @@ -1,5 +1,12 @@ +from __future__ import annotations + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import iluvatar as cdi_iluvatar +from gpustack_runtime.deployer.cdi.iluvatar import IluvatarGenerator +from gpustack_runtime.detector import Device, ManufacturerEnum, iluvatar +from gpustack_runtime.detector.__utils__ import byte_to_mebibyte from gpustack_runtime.detector.iluvatar import IluvatarDetector @@ -21,3 +28,306 @@ def test_get_topology(): det = IluvatarDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pyixml carrying a call log, so a test can assert exactly which # +# driver calls detect_info/detect_usage make -- several acceptance criteria # +# are "issues no metric call", which the returned values alone cannot prove. # +# --------------------------------------------------------------------------- # + + +class _FakeNVMLError(Exception): + """ + Stand-in for pyixml.NVMLError. + """ + + +class _FakeHandle: + def __init__(self, index: int): + self.index = index + + +class _FakeMemoryInfo: + def __init__(self, total: int, used: int): + self.total = total + self.used = used + + +class _FakeUtilizationRates: + def __init__(self, gpu: int): + self.gpu = gpu + + +class _FakePciInfo: + def __init__(self, bus_id: str): + self.busIdLegacy = bus_id + + +class FakePyixml: + """ + A fake pyixml binding, standing in for the real ctypes module so the + Iluvatar detector can be exercised on a machine with no IXML driver. + """ + + NVMLError = _FakeNVMLError + IXML_HEALTH_OK = 0 + NVML_TEMPERATURE_GPU = 0 + NVML_AFFINITY_SCOPE_NODE = 0 + nvmlMemory_v2 = 0x02000028 # noqa: N815 + + def __init__(self, *, device_count: int = 2, v2_memory: bool = True): + self.calls: list[str] = [] + self.device_count = device_count + self.v2_memory = v2_memory + # Deliberately different from the v1 numbers, so a test can tell + # which accessor a returned value came from. + self.v2_memory_total = 32 * 1024**3 + self.v2_memory_used = 8 * 1024**3 + self.v1_memory_total = 16 * 1024**3 + self.v1_memory_used = 4 * 1024**3 + + # The method names below mirror pyixml's real (camelCase) API one-for-one, + # so a test can monkeypatch this in as a drop-in for the module. + def nvmlInit(self): # noqa: N802 + self.calls.append("nvmlInit") + + def nvmlSystemGetDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetDriverVersion") + return "4.2.0" + + def nvmlSystemGetCudaDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetCudaDriverVersion") + return 10020 + + def nvmlDeviceGetCount(self): # noqa: N802 + self.calls.append("nvmlDeviceGetCount") + return self.device_count + + def nvmlDeviceGetHandleByIndex(self, index): # noqa: N802 + self.calls.append("nvmlDeviceGetHandleByIndex") + return _FakeHandle(index) + + def nvmlDeviceGetMinorNumber(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetMinorNumber") + return 10 + dev.index + + def nvmlDeviceGetName(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetName") + return "Iluvatar BI-V150" + + def nvmlDeviceGetUUID(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetUUID") + return f"GPU-{dev.index}" + + def nvmlDeviceGetNumGpuCores(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetNumGpuCores") + return 4096 + + def nvmlDeviceGetMemoryInfo(self, dev, version=None): # noqa: N802 + if version: + self.calls.append("nvmlDeviceGetMemoryInfo_v2") + if not self.v2_memory: + msg = "v2 memory info not supported" + raise self.NVMLError(msg) + return _FakeMemoryInfo(self.v2_memory_total, self.v2_memory_used) + self.calls.append("nvmlDeviceGetMemoryInfo") + return _FakeMemoryInfo(self.v1_memory_total, self.v1_memory_used) + + def ixmlDeviceGetHealth(self, dev): # noqa: N802 + self.calls.append("ixmlDeviceGetHealth") + return self.IXML_HEALTH_OK + + def nvmlDeviceGetPowerManagementDefaultLimit(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerManagementDefaultLimit") + return 250_000 + + def nvmlDeviceGetPowerUsage(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerUsage") + return 90_000 + + def nvmlDeviceGetCudaComputeCapability(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetCudaComputeCapability") + return (7, 0) + + def nvmlDeviceGetPciInfo(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPciInfo") + return _FakePciInfo(f"0000:0{dev.index}:00.0") + + def nvmlDeviceGetMemoryAffinity(self, dev, size, scope): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryAffinity") + return [0] + + def nvmlDeviceGetUtilizationRates(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetUtilizationRates") + return _FakeUtilizationRates(37) + + def nvmlDeviceGetTemperature(self, dev, sensor): # noqa: N802 + self.calls.append("nvmlDeviceGetTemperature") + return 55 + + +@pytest.fixture(autouse=True) +def _reset_is_supported_cache(): + # is_supported()/detect_pci_devices() are lru_cache'd, so a value + # observed by one test would otherwise leak into the next. + IluvatarDetector.is_supported.cache_clear() + IluvatarDetector.detect_pci_devices.cache_clear() + yield + IluvatarDetector.is_supported.cache_clear() + IluvatarDetector.detect_pci_devices.cache_clear() + + +@pytest.fixture +def fake_pyixml(monkeypatch): + def _install(**kwargs) -> FakePyixml: + fake = FakePyixml(**kwargs) + monkeypatch.setattr(iluvatar, "pyixml", fake) + # No PCI sysfs tree exists on the dev machine, so bypass the PCI + # presence check that is_supported() otherwise gates on. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: memory V2->V1 fallback, no vgpu, no usage calls. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_v2_memory_over_v1(fake_pyixml): + fake = fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert [dev.memory for dev in devices] == [ + byte_to_mebibyte(fake.v2_memory_total), + ] * fake.device_count + assert "nvmlDeviceGetMemoryInfo_v2" in fake.calls + assert "nvmlDeviceGetMemoryInfo" not in fake.calls + + +def test_detect_info_falls_back_to_v1_memory_when_v2_unavailable(fake_pyixml): + fake = fake_pyixml(v2_memory=False) + + devices = IluvatarDetector().detect_info() + + assert [dev.memory for dev in devices] == [ + byte_to_mebibyte(fake.v1_memory_total), + ] * fake.device_count + assert fake.calls.count("nvmlDeviceGetMemoryInfo_v2") == fake.device_count + assert fake.calls.count("nvmlDeviceGetMemoryInfo") == fake.device_count + + +def test_detect_info_has_no_vgpu_in_appendix(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert all("vgpu" not in dev.appendix for dev in devices) + + +def test_detect_info_keeps_the_minor_number_appendix(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert [dev.appendix["minor_number"] for dev in devices] == [10, 11] + + +def test_detect_info_issues_no_usage_calls(fake_pyixml): + fake = fake_pyixml() + + IluvatarDetector().detect_info() + + usage_only_calls = { + "nvmlDeviceGetUtilizationRates", + "nvmlDeviceGetTemperature", + "nvmlDeviceGetPowerUsage", + } + assert usage_only_calls.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# detect_usage: merges by uuid, same V2->V1 fallback. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_the_usage_fields_by_uuid(fake_pyixml): + fake = fake_pyixml() + + devices = IluvatarDetector().detect_info() + fake.calls.clear() + + result = IluvatarDetector().detect_usage(devices) + + assert result is devices + for dev in devices: + assert dev.cores_utilization == 37 + assert dev.memory_used == byte_to_mebibyte(fake.v2_memory_used) + assert dev.temperature == 55 + assert dev.power_used == 90 + + +def test_detect_usage_falls_back_to_v1_memory_when_v2_unavailable(fake_pyixml): + fake = fake_pyixml(v2_memory=False) + + devices = IluvatarDetector().detect_info() + result = IluvatarDetector().detect_usage(devices) + + assert all( + dev.memory_used == byte_to_mebibyte(fake.v1_memory_used) for dev in result + ) + + +def test_detect_composes_info_and_usage_by_default(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect() + + assert devices[0].cores_utilization == 37 + assert devices[0].power_used == 90 + assert "vgpu" not in devices[0].appendix + + +# --------------------------------------------------------------------------- # +# CDI: /dev/iluvatar{N} still reads the appendix minor number. # +# --------------------------------------------------------------------------- # + + +def test_cdi_reads_the_appendix_minor_number(monkeypatch): + seen_paths: list[str] = [] + + def _fake_device_node(path): + seen_paths.append(path) + return {"path": path} + + monkeypatch.setattr(cdi_iluvatar, "device_to_cdi_device_node", _fake_device_node) + + devices = [ + Device( + manufacturer=ManufacturerEnum.ILUVATAR, + index=0, + name="Iluvatar BI-V150", + uuid="GPU-0", + memory=32768, + appendix={"bdf": "0000:00:00.0", "minor_number": 7}, + ), + Device( + manufacturer=ManufacturerEnum.ILUVATAR, + index=1, + name="Iluvatar BI-V150", + uuid="GPU-1", + memory=32768, + appendix={"bdf": "0000:01:00.0"}, + ), + ] + + config = IluvatarGenerator().generate(devices) + + assert config is not None + # The device carrying a minor number is addressed by it. + assert "/dev/iluvatar7" in seen_paths + # The device without one falls back to Device.index. + assert "/dev/iluvatar1" in seen_paths From 7e74da5a98c3ff202822330b14614c06ae8f1c4e Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:43 +0800 Subject: [PATCH 08/12] fix(detector): stop dropping every metax card on a virtualization-enabled host - delete the skip on MXSML_VIRTUALIZATION_MODE_PF, which dropped the physical function -- the whole card -- so such a host reported no devices at all - apply no virtualization-mode filter at all, reporting whatever the driver enumerates, and drop the `vgpu` appendix key - move core utilization, used memory, temperature and board power into `detect_usage`, keeping the power limit in the inventory query - cover all three virtualization modes, so a reintroduced filter fails the suite Signed-off-by: thxCode --- gpustack_runtime/detector/metax.py | 187 ++++++--- tests/gpustack_runtime/detector/test_metax.py | 367 ++++++++++++++++++ 2 files changed, 505 insertions(+), 49 deletions(-) diff --git a/gpustack_runtime/detector/metax.py b/gpustack_runtime/detector/metax.py index ebd581c..55508e1 100644 --- a/gpustack_runtime/detector/metax.py +++ b/gpustack_runtime/detector/metax.py @@ -16,6 +16,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -90,9 +91,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.METAX) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect MetaX GPUs using pymtml. + Detect MetaX GPUs' inventory using pymxsml, without usage metrics. Returns: A list of detected MetaX GPU devices, @@ -122,62 +123,31 @@ def detect(self) -> Devices | None: pymxsml.MXSML_VERSION_DRIVER, ) + # No virtualization-mode filter is applied: whatever the driver + # enumerates is reported. This used to skip + # MXSML_VIRTUALIZATION_MODE_PF, dropping the physical function, + # i.e. the whole card, so a virtualization-enabled host came up + # with zero devices. The operator drops VF instead; reporting + # every mode is a deliberate divergence from it. dev_info = pymxsml.mxSmlGetDeviceInfo(dev_idx) dev_uuid = dev_info.uuid dev_name = dev_info.deviceName - if dev_info.mode == pymxsml.MXSML_VIRTUALIZATION_MODE_PF: - continue - - dev_core_util = pymxsml.mxSmlGetDeviceIpUsage( - dev_idx, - pymxsml.MXSML_USAGE_XCORE, - ) - if dev_core_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_core_util = 0 dev_mem_info = pymxsml.mxSmlGetMemoryInfo(dev_idx) dev_mem = kibibyte_to_mebibyte( # KiB to MiB dev_mem_info.vramTotal, ) - dev_mem_used = kibibyte_to_mebibyte( # KiB to MiB - dev_mem_info.vramUse, - ) - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pymxsml.MXSMLError): - dev_ecc_errors = pymxsml.mxSmlGetTotalEccErrors(dev_idx) - if dev_ecc_errors.dramUE > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_temp = ( - pymxsml.mxSmlGetTemperatureInfo( - dev_idx, - pymxsml.MXSML_TEMPERATURE_HOTSPOT, - ) - // 100 # mC to C - ) + dev_mem_status = _get_memory_status(dev_idx) + # The board power *limit* is inventory, unlike the board power + # *usage*, which the usage query reads. dev_power = ( pymxsml.mxSmlGetBoardPowerLimit(dev_idx) // 1000 # mW to W ) - dev_power_used = None - dev_power_info = pymxsml.mxSmlGetBoardPowerInfo(dev_idx) - if dev_power_info: - dev_power_used = ( - sum(i.power if i.power else 0 for i in dev_power_info) - // 1000 # mW to W - ) dev_bdf = dev_info.bdfId dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf) - dev_is_vgpu = dev_info.mode == pymxsml.MXSML_VIRTUALIZATION_MODE_VF - dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: with contextlib.suppress(pymxsml.MXSMLError): @@ -188,7 +158,6 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -207,14 +176,9 @@ def detect(self) -> Devices | None: driver_version=dev_driver_ver, runtime_version=sys_runtime_ver, runtime_version_original=sys_runtime_ver_original, - cores_utilization=dev_core_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -228,6 +192,103 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of MetaX GPUs using pymxsml, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pymxsml.mxSmlInit() + + dev_count = pymxsml.mxSmlGetDeviceCount() + for dev_idx in range(dev_count): + # MXSML enumerates by index and offers no lookup by UUID, so the + # whole set is read and merge_devices_usage keeps what the caller + # asked for, as the operator's MonitorAccelerator does. + dev_info = pymxsml.mxSmlGetDeviceInfo(dev_idx) + dev_uuid = dev_info.uuid + + dev_core_util = pymxsml.mxSmlGetDeviceIpUsage( + dev_idx, + pymxsml.MXSML_USAGE_XCORE, + ) + if dev_core_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_core_util = 0 + + dev_mem_info = pymxsml.mxSmlGetMemoryInfo(dev_idx) + dev_mem = kibibyte_to_mebibyte( # KiB to MiB + dev_mem_info.vramTotal, + ) + dev_mem_used = kibibyte_to_mebibyte( # KiB to MiB + dev_mem_info.vramUse, + ) + dev_mem_status = _get_memory_status(dev_idx) + + dev_temp = ( + pymxsml.mxSmlGetTemperatureInfo( + dev_idx, + pymxsml.MXSML_TEMPERATURE_HOTSPOT, + ) + // 100 # mC to C + ) + + dev_power_used = None + dev_power_info = pymxsml.mxSmlGetBoardPowerInfo(dev_idx) + if dev_power_info: + dev_power_used = ( + sum(i.power if i.power else 0 for i in dev_power_info) + // 1000 # mW to W + ) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_core_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pymxsml.MXSMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -242,7 +303,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -293,6 +354,34 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_status(dev_idx: int) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + Args: + dev_idx: + The device index. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pymxsml.MXSMLError): + dev_ecc_errors = pymxsml.mxSmlGetTotalEccErrors(dev_idx) + if dev_ecc_errors.dramUE > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_card_and_renderd_id(dev_bdf: str) -> tuple[int | None, int | None]: """ Get the card ID and renderD ID for a given device bdf. diff --git a/tests/gpustack_runtime/detector/test_metax.py b/tests/gpustack_runtime/detector/test_metax.py index d86bb5c..6681aa3 100644 --- a/tests/gpustack_runtime/detector/test_metax.py +++ b/tests/gpustack_runtime/detector/test_metax.py @@ -1,5 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import metax, pymxsml +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.metax import MetaXDetector @@ -21,3 +31,360 @@ def test_get_topology(): det = MetaXDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pymxsml, so the split is provable on a host with no MetaX driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MetaX C500" + mode: int = pymxsml.MXSML_VIRTUALIZATION_MODE_NONE + bdf: str = "0000:01:00.0" + driver_version: str = "2.23.0.1" + vram_total: int = 67108864 # KiB, i.e. 65536 MiB + vram_used: int = 1048576 # KiB, i.e. 1024 MiB + dram_ue: int = 0 + core_usage: int = 42 + temperature: int = 5500 + power_limit: int = 350000 # mW, i.e. 350 W + power_ways: tuple[int, ...] = (120000, 30000) # mW, i.e. 150 W together + node_affinity: tuple[int, ...] = (0b1,) # NUMA node 0 + + +@dataclass +class _DeviceInfo: + """ + A c_mxsmlDeviceInfo_t stand-in, carrying the fields the detector reads. + """ + + uuid: str + deviceName: str # noqa: N815 + mode: int + bdfId: str # noqa: N815 + + +@dataclass +class _MemoryInfo: + """ + A c_mxsmlMemoryInfo_t stand-in, in KiB as the driver reports it. + """ + + vramTotal: int # noqa: N815 + vramUse: int # noqa: N815 + + +@dataclass +class _EccErrorCount: + """ + A c_mxSmlEccErrorCount_t stand-in, of which only the uncorrectable DRAM + errors decide the memory health. + """ + + dramUE: int # noqa: N815 + + +@dataclass +class _BoardWayElectricInfo: + """ + A c_mxSmlBoardWayElectricInfo_t stand-in, in mW as the driver reports it. + """ + + power: int + + +@dataclass +class _FakeMXSML: + """ + A stand-in for the pymxsml binding, recording every call the detector makes. + + The error type and the enumeration constants are the real module's, so a + fake drifting from the binding's contract fails here rather than on + hardware. Entry points are dispatched by name, as the fake libcndev.so of + test_pycndev.py does, which keeps the driver's camelCase out of the + handlers' own names. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + MXSMLError = pymxsml.MXSMLError + MXSML_VERSION_DRIVER = pymxsml.MXSML_VERSION_DRIVER + MXSML_USAGE_XCORE = pymxsml.MXSML_USAGE_XCORE + MXSML_TEMPERATURE_HOTSPOT = pymxsml.MXSML_TEMPERATURE_HOTSPOT + + def __getattr__(self, name: str): + handler = { + "mxSmlInit": self._init, + "mxSmlGetMacaVersion": self._get_maca_version, + "mxSmlGetDeviceCount": self._get_device_count, + "mxSmlGetDeviceVersion": self._get_device_version, + "mxSmlGetDeviceInfo": self._get_device_info, + "mxSmlGetMemoryInfo": self._get_memory_info, + "mxSmlGetTotalEccErrors": self._get_total_ecc_errors, + "mxSmlGetBoardPowerLimit": self._get_board_power_limit, + "mxSmlGetNodeAffinity": self._get_node_affinity, + "mxSmlGetDeviceIpUsage": self._get_device_ip_usage, + "mxSmlGetTemperatureInfo": self._get_temperature_info, + "mxSmlGetBoardPowerInfo": self._get_board_power_info, + }.get(name) + if handler is None: + msg = f"module pymxsml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _init(self) -> None: + pass + + def _get_maca_version(self) -> str: + return "2.33.0.6" + + def _get_device_count(self) -> int: + return len(self.cards) + + def _get_device_version(self, device_id: int, version_unit: int) -> str: + assert version_unit == pymxsml.MXSML_VERSION_DRIVER + return self.cards[device_id].driver_version + + def _get_device_info(self, device_id: int) -> _DeviceInfo: + card = self.cards[device_id] + return _DeviceInfo( + uuid=card.uuid, + deviceName=card.name, + mode=card.mode, + bdfId=card.bdf, + ) + + def _get_memory_info(self, device_id: int) -> _MemoryInfo: + card = self.cards[device_id] + return _MemoryInfo(vramTotal=card.vram_total, vramUse=card.vram_used) + + def _get_total_ecc_errors(self, device_id: int) -> _EccErrorCount: + return _EccErrorCount(dramUE=self.cards[device_id].dram_ue) + + def _get_board_power_limit(self, device_id: int) -> int: + return self.cards[device_id].power_limit + + def _get_node_affinity(self, device_id: int, node_set_size: int) -> list[int]: + assert node_set_size > 0 + return list(self.cards[device_id].node_affinity) + + def _get_device_ip_usage(self, device_id: int, usage_ip: int) -> int: + assert usage_ip == pymxsml.MXSML_USAGE_XCORE + return self.cards[device_id].core_usage + + def _get_temperature_info(self, device_id: int, temperature_type: int) -> int: + assert temperature_type == pymxsml.MXSML_TEMPERATURE_HOTSPOT + return self.cards[device_id].temperature + + def _get_board_power_info(self, device_id: int) -> list[_BoardWayElectricInfo]: + return [ + _BoardWayElectricInfo(power=power) + for power in self.cards[device_id].power_ways + ] + + +_USAGE_CALLS = ( + "mxSmlGetDeviceIpUsage", + "mxSmlGetTemperatureInfo", + "mxSmlGetBoardPowerInfo", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: mxSmlGetBoardPowerLimit +reads the power *limit* and mxSmlGetMemoryInfo the memory *total*, both of which +are inventory. +""" + +_MEMORY_UTILIZATION = 1.56 +""" +A default card's memory utilization: 1024 MiB used of 65536 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch): + """ + Build a MetaX detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card) -> tuple[MetaXDetector, _FakeMXSML]: + fake = _FakeMXSML(cards=list(cards)) + monkeypatch.setattr(metax, "pymxsml", fake) + det = MetaXDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: no virtualization-mode filter, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_every_virtualization_mode(detector): + det, _ = detector( + _Card(uuid="GPU-bare", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_NONE), + _Card(uuid="GPU-pf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_PF), + _Card(uuid="GPU-vf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_VF), + ) + + devices = det.detect_info() + + # The PF row is the regression guard: the detector used to `continue` on + # MXSML_VIRTUALIZATION_MODE_PF, i.e. drop the physical function -- the whole + # card -- so a virtualization-enabled host came up with zero devices. The VF + # row pins the other half: no virtualization-mode filter is applied at all, + # which is a deliberate divergence from the operator, as it still drops VF. + assert [dev.uuid for dev in devices] == ["GPU-bare", "GPU-pf", "GPU-vf"] + assert [dev.index for dev in devices] == [0, 1, 2] + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.METAX + assert dev.name == "MetaX C500" + assert dev.uuid == "GPU-0" + assert dev.driver_version == "2.23.0.1" + assert dev.runtime_version == "2.33" + assert dev.runtime_version_original == "2.33.0.6" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["numa"] == "0" + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="GPU-0")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + assert "mxSmlGetBoardPowerLimit" in fake.calls + assert "mxSmlGetMemoryInfo" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="GPU-pf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_PF), + _Card(uuid="GPU-vf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_VF), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="GPU-0"), + _Card( + uuid="GPU-1", + core_usage=7, + vram_used=2097152, + temperature=6100, + power_ways=(200000,), + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["GPU-0"].cores_utilization == 42 + assert by_uuid["GPU-0"].memory_used == 1024 + assert by_uuid["GPU-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["GPU-0"].temperature == 55 + assert by_uuid["GPU-0"].power_used == 150 + assert by_uuid["GPU-1"].cores_utilization == 7 + assert by_uuid["GPU-1"].memory_used == 2048 + assert by_uuid["GPU-1"].temperature == 61 + assert by_uuid["GPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["GPU-0"].index == 0 + assert by_uuid["GPU-0"].memory == 65536 + assert by_uuid["GPU-0"].power == 350 + assert by_uuid["GPU-0"].name == "MetaX C500" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].name == "MetaX C500" + assert devices[0].memory == 65536 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect()[0] + + assert dev.name == "MetaX C500" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reports_an_uncorrectable_ecc_error(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector(_Card(uuid="GPU-0", dram_ue=3)) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY From 2dbbc995c58fc2cb9769c0659c391f3a6d9905e4 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:51:50 +0800 Subject: [PATCH 09/12] fix(detector): stop crashing mthreads detection on a virtualization-capable card - delete the virtRole skip: it read `mpcCap`, which `c_mtmlDeviceProperty_t` does not have, so a card reporting HOST_VIRTDEVICE raised AttributeError and failed the whole detect pass -- the worker saw no MThreads devices at all - apply no virtRole filter, reporting whatever the driver enumerates, and drop the `vgpu` appendix key - move utilization, temperature, used memory and used power into `detect_usage` - cover every virtRole, so a reintroduced filter fails the suite loudly Signed-off-by: thxCode --- gpustack_runtime/detector/mthreads.py | 187 +++++--- .../detector/test_mthreads.py | 422 ++++++++++++++++++ 2 files changed, 559 insertions(+), 50 deletions(-) diff --git a/gpustack_runtime/detector/mthreads.py b/gpustack_runtime/detector/mthreads.py index 0680617..0c6a764 100644 --- a/gpustack_runtime/detector/mthreads.py +++ b/gpustack_runtime/detector/mthreads.py @@ -14,6 +14,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -86,9 +87,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.MTHREADS) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect MThreads GPUs using pymtml. + Detect MThreads GPUs' inventory using pymtml, without usage metrics. Returns: A list of detected MThreads GPU devices, @@ -115,62 +116,34 @@ def detect(self) -> Devices | None: dev_uuid = "" dev_name = "" dev_cores = 0 - dev_power_used = None dev_pci_info = None - dev_is_vgpu = False dev = pymtml.mtmlLibraryInitDeviceByIndex(dev_idx) try: - dev_props = pymtml.mtmlDeviceGetProperty(dev) - dev_is_vgpu = ( - dev_props.virtRole == pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE - ) - if ( - dev_is_vgpu - and dev_props.mpcCap != pymtml.MTML_MPC_TYPE_INSTANCE - ): - continue - + # No virtRole filter is applied: whatever the driver + # enumerates is reported. This used to read the device + # property and skip a MTML_VIRT_ROLE_HOST_VIRTDEVICE card + # whose `mpcCap` was not MTML_MPC_TYPE_INSTANCE -- a field + # c_mtmlDeviceProperty_t does not have, so the read raised + # AttributeError and failed the whole detect pass instead. + # The operator skips GUEST_VIRTDEVICE instead; reporting + # every role is a deliberate divergence from it. dev_uuid = pymtml.mtmlDeviceGetUUID(dev) dev_name = pymtml.mtmlDeviceGetName(dev) dev_cores = pymtml.mtmlDeviceCountGpuCores(dev) - dev_power_used = pymtml.mtmlDeviceGetPowerUsage(dev) dev_pci_info = pymtml.mtmlDeviceGetPciInfo(dev) finally: pymtml.mtmlLibraryFreeDevice(dev) + # MTML binds no power *limit* call, only the power *usage* the + # usage query reads, so the device carries no power limit. + dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + dev_mem_status = DeviceMemoryStatusEnum.UNKNOWN with pymtml.mtmlMemoryContext(dev) as devmem: dev_mem = byte_to_mebibyte( # byte to MiB pymtml.mtmlMemoryGetTotal(devmem), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pymtml.mtmlMemoryGetUsed(devmem), - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pymtml.mtmlMemoryGetEccErrorCounter( - devmem, - pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED, - pymtml.MTML_VOLATILE_ECC, - pymtml.MTML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_cores_util = None - dev_temp = None - with pymtml.mtmlGpuContext(dev) as devgpu: - dev_cores_util = pymtml.mtmlGpuGetUtilization(devgpu) - dev_temp = pymtml.mtmlGpuGetTemperature(devgpu) - - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 + dev_mem_status = _get_memory_status(devmem) dev_bdf = f"{dev_pci_info.segment:04x}:{dev_pci_info.bus:02x}:{dev_pci_info.device:02x}.0" @@ -186,7 +159,6 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -200,13 +172,8 @@ def detect(self) -> Devices | None: name=dev_name, driver_version=sys_driver_ver, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -223,6 +190,91 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of MThreads GPUs using pymtml, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pymtml.mtmlLibraryInit() + + dev_count = pymtml.mtmlLibraryCountDevice() + for dev_idx in range(dev_count): + # MTML enumerates by index, so the whole set is read and + # merge_devices_usage keeps what the caller asked for, + # as the operator's MonitorAccelerator does. + dev = pymtml.mtmlLibraryInitDeviceByIndex(dev_idx) + try: + dev_uuid = pymtml.mtmlDeviceGetUUID(dev) + dev_power_used = pymtml.mtmlDeviceGetPowerUsage(dev) + + with pymtml.mtmlMemoryContext(dev) as devmem: + dev_mem = byte_to_mebibyte( # byte to MiB + pymtml.mtmlMemoryGetTotal(devmem), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pymtml.mtmlMemoryGetUsed(devmem), + ) + dev_mem_status = _get_memory_status(devmem) + + with pymtml.mtmlGpuContext(dev) as devgpu: + dev_cores_util = pymtml.mtmlGpuGetUtilization(devgpu) + dev_temp = pymtml.mtmlGpuGetTemperature(devgpu) + finally: + pymtml.mtmlLibraryFreeDevice(dev) + + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pymtml.MTMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between MThreads GPUs. @@ -237,7 +289,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -315,6 +367,41 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_status( + devmem: pymtml.c_mtmlMemory_t, +) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + Args: + devmem: + The device memory handle. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pymtml.MTMLError): + dev_mem_ecc_errors = pymtml.mtmlMemoryGetEccErrorCounter( + devmem, + pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED, + pymtml.MTML_VOLATILE_ECC, + pymtml.MTML_MEMORY_LOCATION_DRAM, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_links_state( dev: pymtml.c_mtmlDevice_t, ) -> dict | None: diff --git a/tests/gpustack_runtime/detector/test_mthreads.py b/tests/gpustack_runtime/detector/test_mthreads.py index 59170b6..45d51d7 100644 --- a/tests/gpustack_runtime/detector/test_mthreads.py +++ b/tests/gpustack_runtime/detector/test_mthreads.py @@ -1,5 +1,16 @@ +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import mthreads, pymtml +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.mthreads import MThreadsDetector @@ -21,3 +32,414 @@ def test_get_topology(): det = MThreadsDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pymtml, so the split is provable on a host with no MThreads driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MTT S4000" + virt_role: int = pymtml.MTML_VIRT_ROLE_NONE + mpc_type: int = pymtml.MTML_MPC_TYPE_NONE + bus: int = 0x01 + cores: int = 128 + memory_total: int = 51539607552 # byte, i.e. 49152 MiB + memory_used: int = 1073741824 # byte, i.e. 1024 MiB + memory_ecc_dram_ue: int = 0 + cores_utilization: int = 42 + temperature: int = 55 + power_usage: int = 150 + node_affinity: tuple[int, ...] = (0b1,) # NUMA node 0 + + +@dataclass +class _DeviceProperty: + """ + A c_mtmlDeviceProperty_t stand-in. + + Its real field names, deliberately: the removed filter read a `mpcCap` + field that the struct does not have -- it carries `mpcCapability` and + `mpcType` -- so on a card reporting virtRole HOST_VIRTDEVICE the read raised + AttributeError and failed the whole detect pass. + """ + + virtCapability: int # noqa: N815 + virtRole: int # noqa: N815 + mpcCapability: int # noqa: N815 + mpcType: int # noqa: N815 + + +@dataclass +class _PciInfo: + """ + A c_mtmlPciInfo_t stand-in, carrying the fields the detector reads. + """ + + segment: int + bus: int + device: int + + +@dataclass +class _Handle: + """ + A device/memory/GPU handle stand-in, carrying the card it belongs to. + """ + + index: int + + +@dataclass +class _FakeMTML: + """ + A stand-in for the pymtml binding, recording every call the detector makes. + + The error type and the enumeration constants the detector legitimately needs + are the real module's, so a fake drifting from the binding's contract fails + here rather than on hardware. The virtRole and MPC constants are left out on + purpose: a reintroduced virtRole filter then fails loudly with AttributeError + instead of quietly reporting fewer cards. Entry points are dispatched by + name, which keeps the driver's camelCase out of the handlers' own names. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + MTMLError = pymtml.MTMLError + MTML_MEMORY_ERROR_TYPE_UNCORRECTED = pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED + MTML_VOLATILE_ECC = pymtml.MTML_VOLATILE_ECC + MTML_MEMORY_LOCATION_DRAM = pymtml.MTML_MEMORY_LOCATION_DRAM + + def __getattr__(self, name: str): + handler = { + "mtmlLibraryInit": self._library_init, + "mtmlLibraryInitSystem": self._library_init_system, + "mtmlLibraryFreeSystem": self._library_free_system, + "mtmlLibraryCountDevice": self._library_count_device, + "mtmlLibraryInitDeviceByIndex": self._library_init_device_by_index, + "mtmlLibraryFreeDevice": self._library_free_device, + "mtmlSystemGetDriverVersion": self._system_get_driver_version, + "mtmlDeviceGetProperty": self._device_get_property, + "mtmlDeviceGetUUID": self._device_get_uuid, + "mtmlDeviceGetName": self._device_get_name, + "mtmlDeviceCountGpuCores": self._device_count_gpu_cores, + "mtmlDeviceGetPciInfo": self._device_get_pci_info, + "mtmlDeviceGetPowerUsage": self._device_get_power_usage, + "mtmlDeviceGetMemoryAffinityWithinNode": self._device_get_memory_affinity, + "mtmlMemoryContext": self._memory_context, + "mtmlMemoryGetTotal": self._memory_get_total, + "mtmlMemoryGetUsed": self._memory_get_used, + "mtmlMemoryGetEccErrorCounter": self._memory_get_ecc_error_counter, + "mtmlGpuContext": self._gpu_context, + "mtmlGpuGetUtilization": self._gpu_get_utilization, + "mtmlGpuGetTemperature": self._gpu_get_temperature, + }.get(name) + if handler is None: + msg = f"module pymtml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _library_init(self) -> None: + pass + + def _library_init_system(self) -> object: + return object() + + def _library_free_system(self, system) -> None: + pass + + def _library_count_device(self) -> int: + return len(self.cards) + + def _library_init_device_by_index(self, index: int) -> _Handle: + return _Handle(index=index) + + def _library_free_device(self, device: _Handle) -> None: + pass + + def _system_get_driver_version(self, system) -> str: + return "2.7.0" + + def _device_get_property(self, device: _Handle) -> _DeviceProperty: + card = self.cards[device.index] + return _DeviceProperty( + virtCapability=0, + virtRole=card.virt_role, + mpcCapability=0, + mpcType=card.mpc_type, + ) + + def _device_get_uuid(self, device: _Handle) -> str: + return self.cards[device.index].uuid + + def _device_get_name(self, device: _Handle) -> str: + return self.cards[device.index].name + + def _device_count_gpu_cores(self, device: _Handle) -> int: + return self.cards[device.index].cores + + def _device_get_pci_info(self, device: _Handle) -> _PciInfo: + return _PciInfo(segment=0, bus=self.cards[device.index].bus, device=0) + + def _device_get_power_usage(self, device: _Handle) -> int: + return self.cards[device.index].power_usage + + def _device_get_memory_affinity( + self, + device: _Handle, + node_set_size: int, + ) -> list[int]: + assert node_set_size > 0 + return list(self.cards[device.index].node_affinity) + + def _memory_context(self, device: _Handle): + return contextlib.nullcontext(_Handle(index=device.index)) + + def _memory_get_total(self, memory: _Handle) -> int: + return self.cards[memory.index].memory_total + + def _memory_get_used(self, memory: _Handle) -> int: + return self.cards[memory.index].memory_used + + def _memory_get_ecc_error_counter( + self, + memory: _Handle, + error_type: int, + counter_type: int, + location_type: int, + ) -> int: + assert error_type == pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED + assert counter_type == pymtml.MTML_VOLATILE_ECC + assert location_type == pymtml.MTML_MEMORY_LOCATION_DRAM + return self.cards[memory.index].memory_ecc_dram_ue + + def _gpu_context(self, device: _Handle): + return contextlib.nullcontext(_Handle(index=device.index)) + + def _gpu_get_utilization(self, gpu: _Handle) -> int: + return self.cards[gpu.index].cores_utilization + + def _gpu_get_temperature(self, gpu: _Handle) -> int: + return self.cards[gpu.index].temperature + + +_USAGE_CALLS = ( + "mtmlGpuContext", + "mtmlGpuGetUtilization", + "mtmlGpuGetTemperature", + "mtmlDeviceGetPowerUsage", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: mtmlMemoryGetTotal +reads the memory *total*, which is inventory. MTML binds no power *limit* call +at all, so nothing of the sort stays behind in the information query. +""" + +_MEMORY_UTILIZATION = 2.08 +""" +A default card's memory utilization: 1024 MiB used of 49152 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch): + """ + Build an MThreads detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card) -> tuple[MThreadsDetector, _FakeMTML]: + fake = _FakeMTML(cards=list(cards)) + monkeypatch.setattr(mthreads, "pymtml", fake) + det = MThreadsDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: no virtRole filter, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_every_virt_role(detector): + det, _ = detector( + _Card(uuid="GPU-bare", virt_role=pymtml.MTML_VIRT_ROLE_NONE), + _Card( + uuid="GPU-host-virt", + virt_role=pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE, + mpc_type=pymtml.MTML_MPC_TYPE_PARENT, # i.e. != MPC_TYPE_INSTANCE + ), + _Card( + uuid="GPU-guest-virt", + virt_role=pymtml.MTML_VIRT_ROLE_GUEST_VIRTDEVICE, + ), + ) + + devices = det.detect_info() + + # The host-virt row is the regression guard: the detector used to read a + # `mpcCap` field that c_mtmlDeviceProperty_t does not carry, so such a card + # raised AttributeError and failed the whole detect pass -- the worker saw + # zero MThreads devices, not merely a missing one. The guest-virt row pins + # the other half: no virtRole filter is applied at all, which is a deliberate + # divergence from the operator, as it still drops GUEST_VIRTDEVICE. + assert [dev.uuid for dev in devices] == [ + "GPU-bare", + "GPU-host-virt", + "GPU-guest-virt", + ] + assert [dev.index for dev in devices] == [0, 1, 2] + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.MTHREADS + assert dev.name == "MTT S4000" + assert dev.uuid == "GPU-0" + assert dev.driver_version == "2.7.0" + assert dev.cores == 128 + assert dev.memory == 49152 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["numa"] == "0" + # MTML binds no power *limit* call, so the field stays unset rather than + # borrowing the power usage the usage query reads. + assert dev.power is None + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="GPU-0")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # The property read went away with the filter it served, and nothing else + # consumes it. + assert "mtmlDeviceGetProperty" not in fake.calls + assert "mtmlMemoryGetTotal" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="GPU-host-virt", virt_role=pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE), + _Card(uuid="GPU-guest-virt", virt_role=pymtml.MTML_VIRT_ROLE_GUEST_VIRTDEVICE), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="GPU-0"), + _Card( + uuid="GPU-1", + cores_utilization=7, + memory_used=2147483648, # byte, i.e. 2048 MiB + temperature=61, + power_usage=200, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["GPU-0"].cores_utilization == 42 + assert by_uuid["GPU-0"].memory_used == 1024 + assert by_uuid["GPU-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["GPU-0"].temperature == 55 + assert by_uuid["GPU-0"].power_used == 150 + assert by_uuid["GPU-1"].cores_utilization == 7 + assert by_uuid["GPU-1"].memory_used == 2048 + assert by_uuid["GPU-1"].temperature == 61 + assert by_uuid["GPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["GPU-0"].index == 0 + assert by_uuid["GPU-0"].memory == 49152 + assert by_uuid["GPU-0"].cores == 128 + assert by_uuid["GPU-0"].name == "MTT S4000" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].name == "MTT S4000" + assert devices[0].memory == 49152 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect()[0] + + assert dev.name == "MTT S4000" + assert dev.memory == 49152 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reports_an_uncorrectable_ecc_error(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector(_Card(uuid="GPU-0", memory_ecc_dram_ue=3)) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY From ee6964ba335e8026f675d0f40f7e529a7b034315 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:52:00 +0800 Subject: [PATCH 10/12] refactor(detector): split thead's usage query and prove its ordinal reaches the card - move utilization, temperature and used power into `detect_usage`, instance entries included, keeping the memory total and the power limit as inventory - suppress a failing MIG read per instance rather than per card, so one instance refusing a read no longer aborts the loop and erases every later instance from the inventory, or leaves it reading as idle - drop the `vgpu` appendix key from the card and its instances, and the SR-IOV physical-function comparison - keep the GPU/compute-instance entries in `appendix["mig_devices"]`, `sliced` intact, since the operator has no equivalent and the topology path reads it - record `appendix["minor_number"]` and make the CDI generator compare it: where the detector read a minor, the node /dev/alixpu_ppu must carry it, or the device is refused. The failure that refuses is the operator's measured one -- a path built from the minor lands on the neighbouring accelerator silently, and on the last card of a 16-card host names a ppu16 that does not exist. Signed-off-by: thxCode --- gpustack_runtime/deployer/cdi/thead.py | 18 + gpustack_runtime/detector/thead.py | 386 ++++++-- tests/gpustack_runtime/detector/test_thead.py | 830 ++++++++++++++++++ 3 files changed, 1139 insertions(+), 95 deletions(-) create mode 100644 tests/gpustack_runtime/detector/test_thead.py diff --git a/gpustack_runtime/deployer/cdi/thead.py b/gpustack_runtime/deployer/cdi/thead.py index c2fa9b3..56bda5f 100644 --- a/gpustack_runtime/deployer/cdi/thead.py +++ b/gpustack_runtime/deployer/cdi/thead.py @@ -82,11 +82,29 @@ def generate( container_device_nodes = [] + # Named after the card ordinal, i.e. the enumeration index, not + # after the driver's minor number that the detector records in + # appendix["minor_number"]. Unlike /dev/nvidia{N} and + # /dev/iluvatar{N}, which the minor number does name, the operator + # records T-Head's purely to PROVE this node addresses the card it + # describes, by comparing it against the node's character-device + # minor. cdn = device_to_cdi_device_node( path=f"/dev/alixpu_ppu{dev.index}", ) if not cdn: continue + + # So make that comparison rather than assume it: where the detector + # read a minor, the node this ordinal names must carry it. The two + # numbers are independent -- neither is computed from the other, at + # any offset or none -- so a mismatch means this ordinal addresses a + # neighbouring accelerator, which the operator's allocator refuses + # outright rather than hand over. + dev_minor_number = dev.appendix.get("minor_number") + if dev_minor_number is not None and cdn.minor != dev_minor_number: + continue + all_device_nodes.append(cdn) container_device_nodes.append(cdn) diff --git a/gpustack_runtime/detector/thead.py b/gpustack_runtime/detector/thead.py index 73b4832..483b100 100644 --- a/gpustack_runtime/detector/thead.py +++ b/gpustack_runtime/detector/thead.py @@ -19,6 +19,7 @@ Topology, TopologyDistanceEnum, index_mig_devices, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -28,7 +29,6 @@ get_numa_node_by_bdf, get_numa_nodeset_size, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) @@ -81,9 +81,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.THEAD) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect T-Head GPUs using pyhgml. + Detect T-Head GPUs' inventory using pyhgml, without usage metrics. Returns: A list of detected T-Head GPU devices, @@ -148,21 +148,12 @@ def detect(self) -> Devices | None: ) dev_numa = bitmask_to_str(list(dev_node_affinity)) - dev_temp = None - with contextlib.suppress(pyhgml.HGMLError): - dev_temp = pyhgml.hgmlDeviceGetTemperature( - dev, - pyhgml.HGML_TEMPERATURE_GPU, - ) - + # The power limit is inventory; the power actually drawn is + # usage, and belongs to detect_usage. dev_power = None - dev_power_used = None with contextlib.suppress(pyhgml.HGMLError): dev_power = pyhgml.hgmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pyhgml.hgmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_mig_mode = pyhgml.HGML_DEVICE_MIG_DISABLE with contextlib.suppress(pyhgml.HGMLError): @@ -170,6 +161,18 @@ def detect(self) -> Devices | None: dev_index = dev_idx + # Device.index is the enumeration index, while the driver's + # minor number goes to the appendix. Unlike NVIDIA's and + # Iluvatar's, the T-Head device node is named after the card + # ordinal and not after this number: the operator records it + # purely to PROVE a node addresses the card it describes, so it + # is left absent when the driver cannot answer rather than + # substituted by the enumeration index -- a substituted value + # would make a wrong ordinal look proven. + dev_minor_number = None + with contextlib.suppress(pyhgml.HGMLError): + dev_minor_number = pyhgml.hgmlDeviceGetMinorNumber(dev) + # Report the physical card, whether or not MIG is enabled. # MIG instances are partitioned on demand by the operator's # device-manager; they are not separate allocatable devices @@ -184,48 +187,19 @@ def detect(self) -> Devices | None: with contextlib.suppress(pyhgml.HGMLError): dev_cores = pyhgml.hgmlDeviceGetNumGpuCores(dev) - dev_cores_util = None - with contextlib.suppress(pyhgml.HGMLError): - dev_util_rates = pyhgml.hgmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pyhgml.HGMLError): - dev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(dev) - dev_mem = byte_to_mebibyte( # byte to MiB - dev_mem_info.total, - ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( - dev, - pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, - pyhgml.HGML_VOLATILE_ECC, - pyhgml.HGML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_is_vgpu = False - if dev_bdf: - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf + dev_mem, _ = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pyhgml.HGML_VOLATILE_ECC, + pyhgml.HGML_MEMORY_LOCATION_DRAM, + ) dev_appendix = { - "vgpu": dev_is_vgpu, "mig": dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE: dev_mig_slots = 0 with contextlib.suppress(pyhgml.HGMLError): @@ -238,9 +212,7 @@ def detect(self) -> Devices | None: sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf, dev_numa, ) @@ -260,14 +232,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -282,6 +249,115 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch T-Head GPUs' usage using pyhgml, merged into the given devices. + + Args: + devices: + The devices to refresh, matched by UUID, GPU/compute instance + entries in ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during fetching. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + # The usage query enumerates the driver's devices on its own and returns + # them keyed by UUID, mirroring the operator's MonitorAccelerator: a + # metrics list is joined by device identity, never by index, as an index + # is not stable across a re-detection. + usages: Devices = [] + + try: + pyhgml.hgmlInit() + + dev_count = pyhgml.hgmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pyhgml.hgmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pyhgml.hgmlDeviceGetUUID(dev) + + dev_cores_util = None + with contextlib.suppress(pyhgml.HGMLError): + dev_util_rates = pyhgml.hgmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem, dev_mem_used = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pyhgml.HGML_VOLATILE_ECC, + pyhgml.HGML_MEMORY_LOCATION_DRAM, + ) + + dev_temp = None + with contextlib.suppress(pyhgml.HGMLError): + dev_temp = pyhgml.hgmlDeviceGetTemperature( + dev, + pyhgml.HGML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pyhgml.HGMLError): + dev_power_used = ( + pyhgml.hgmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + dev_mig_mode = pyhgml.HGML_DEVICE_MIG_DISABLE + with contextlib.suppress(pyhgml.HGMLError): + dev_mig_mode, _ = pyhgml.hgmlDeviceGetMigMode(dev) + if dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE: + dev_mig_slots = 0 + with contextlib.suppress(pyhgml.HGMLError): + dev_mig_slots = pyhgml.hgmlDeviceGetMaxMigDeviceCount(dev) + usages.extend( + _get_mig_usages( + dev, + dev_mig_slots, + dev_temp, + dev_power_used, + ), + ) + except pyhgml.HGMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -296,7 +372,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -380,6 +456,71 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_info( + dev: pyhgml.c_hgmlDevice_t, +) -> tuple[int, int]: + """ + Get a device's total and used memory. + + Args: + dev: + The HGML device handle. + + Returns: + The total and used memory in MiB, both 0 if unreadable. + + """ + with contextlib.suppress(pyhgml.HGMLError): + dev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(dev) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + return 0, 0 + + +def _get_memory_status( + dev: pyhgml.c_hgmlDevice_t, + ecc_counter_type: int, + memory_location: int, +) -> DeviceMemoryStatusEnum: + """ + Get a device's memory health from its uncorrected ECC error counter. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. The usage query + cannot skip it: merging usage overwrites the status, so a status it did not + read would erase the one the information query found. + + Args: + dev: + The HGML device handle. + ecc_counter_type: + The ECC counter type to read, volatile or aggregate. + memory_location: + The memory location to read the counter of. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pyhgml.HGMLError): + dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( + dev, + pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, + ecc_counter_type, + memory_location, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_gpm_metrics( metrics: list[int], dev: pyhgml.c_hgmlDevice_t, @@ -559,53 +700,46 @@ def _get_mig_devices( sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf: str, dev_numa, ) -> list[dict]: """ - Enumerate the card's current MIG devices with the same detail a plain - device carries (profile name, uuid, compute/memory utilization, memory - health, temperature and power), returned as appendix entries of the - physical card rather than standalone devices. Empty when MIG is enabled - but no GPU instances exist yet. + Enumerate the card's current MIG devices with the same inventory detail a + plain device carries (profile name, uuid, cores, total memory and memory + health), returned as appendix entries of the physical card rather than + standalone devices. Empty when MIG is enabled but no GPU instances exist + yet. The operator has no T-Head equivalent of this enumeration; keeping it + is a deliberate divergence. + + An entry keeps a Device's shape, so the fields the usage query owns are + present at a Device's defaults: `_get_mig_usages` fills them. Each entry's `index` is the driver slot the MIG device was found at: index_mig_devices turns it into the device index once every card is detected. """ ret: list[dict] = [] - with contextlib.suppress(pyhgml.HGMLError): - for mdev_idx in range(dev_mig_slots): - mdev = None - with contextlib.suppress(pyhgml.HGMLError): - mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing a read + # used to abort the loop, so every later instance vanished from the + # inventory. An empty slot raises here as well, which is how it is + # skipped. + with contextlib.suppress(pyhgml.HGMLError): + mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) if not mdev: continue mdev_uuid = pyhgml.hgmlDeviceGetUUID(mdev) - mdev_mem = 0 - mdev_mem_used = 0 - mdev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pyhgml.HGMLError): - mdev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(mdev) - mdev_mem = byte_to_mebibyte(mdev_mem_info.total) - mdev_mem_used = byte_to_mebibyte(mdev_mem_info.used) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - mdev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( - mdev, - pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, - pyhgml.HGML_AGGREGATE_ECC, - pyhgml.HGML_MEMORY_LOCATION_SRAM, - ) - if mdev_mem_ecc_errors > 0: - mdev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + mdev_mem, _ = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pyhgml.HGML_AGGREGATE_ECC, + pyhgml.HGML_MEMORY_LOCATION_SRAM, + ) mdev_appendix = { - "vgpu": True, "sliced": True, "mig": True, "bdf": dev_bdf, @@ -618,8 +752,6 @@ def _get_mig_devices( mdev_ci_id = pyhgml.hgmlDeviceGetComputeInstanceId(mdev) mdev_appendix["compute_instance_id"] = mdev_ci_id - mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) - mdev_name = "" mdev_cores = None mdev_gi = pyhgml.hgmlDeviceGetGpuInstanceById(dev, mdev_gi_id) @@ -693,20 +825,84 @@ def _get_mig_devices( "runtime_version_original": sys_runtime_ver_original, "compute_capability": dev_cc, "cores": mdev_cores, - "cores_utilization": mdev_cores_util, + "cores_utilization": 0, "memory": mdev_mem, - "memory_used": mdev_mem_used, - "memory_utilization": get_utilization(mdev_mem_used, mdev_mem), + "memory_used": 0, + "memory_utilization": 0, "memory_status": mdev_mem_status, - "temperature": dev_temp, + "temperature": None, "power": dev_power, - "power_used": dev_power_used, + "power_used": None, "appendix": mdev_appendix, }, ) return ret +def _get_mig_usages( + dev, + dev_mig_slots: int, + dev_temp, + dev_power_used, +) -> Devices: + """ + Fetch the usage of the card's current MIG devices, one UUID-keyed entry per + instance, to merge into the card's `appendix["mig_devices"]`. + + Args: + dev: + The HGML device handle of the card hosting them. + dev_mig_slots: + The number of MIG devices the card can host. + dev_temp: + The card's temperature. + dev_power_used: + The card's used power. + + Returns: + The MIG devices' usage, keyed by UUID. + + """ + ret: Devices = [] + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing its UUID + # or its GPU instance id used to abort the loop, so every later instance + # kept the inventory's defaults -- 0 % and 0 MiB, reported idle while it + # may be running a workload. An empty slot raises here as well, which is + # how it is skipped. + with contextlib.suppress(pyhgml.HGMLError): + mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + if not mdev: + continue + + mdev_uuid = pyhgml.hgmlDeviceGetUUID(mdev) + + mdev_mem, mdev_mem_used = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pyhgml.HGML_AGGREGATE_ECC, + pyhgml.HGML_MEMORY_LOCATION_SRAM, + ) + + mdev_gi_id = pyhgml.hgmlDeviceGetGpuInstanceId(mdev) + mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) + + ret.append( + Device( + uuid=mdev_uuid, + cores_utilization=mdev_cores_util, + memory_used=mdev_mem_used, + memory_utilization=get_utilization(mdev_mem_used, mdev_mem), + memory_status=mdev_mem_status, + # A MIG device reports neither temperature nor power, so it + # carries the card's. + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + return ret + + def _get_gpu_instance_slice(dev_gi_prf_id: int) -> int: """ Get the number of slices for a given GPU Instance Profile ID. diff --git a/tests/gpustack_runtime/detector/test_thead.py b/tests/gpustack_runtime/detector/test_thead.py new file mode 100644 index 0000000..0a27761 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_thead.py @@ -0,0 +1,830 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from types import SimpleNamespace + +import pytest + +from gpustack_runtime.deployer.cdi import thead as cdi_thead +from gpustack_runtime.deployer.cdi.thead import THeadGenerator +from gpustack_runtime.detector import pyhgml, thead +from gpustack_runtime.detector.__types__ import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, +) +from gpustack_runtime.detector.thead import THeadDetector + + +@pytest.mark.skipif( + not THeadDetector.is_supported(), + reason="T-Head PPU not detected", +) +def test_detect(): + det = THeadDetector() + devs = det.detect() + print(devs) + + +@pytest.mark.skipif( + not THeadDetector.is_supported(), + reason="T-Head PPU not detected", +) +def test_get_topology(): + det = THeadDetector() + topo = det.get_topology() + print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pyhgml, so the split is provable on a host with no T-Head driver. # +# --------------------------------------------------------------------------- # + +_MIB = 1 << 20 + + +@dataclass +class _Instance: + """ + One GPU/compute instance of a MIG-enabled card, as the driver reports it. + + The runtime enumerates these where the operator does not: the physical card + stays the reported device and its instances live in + ``appendix["mig_devices"]``. + """ + + uuid: str + gpu_instance_id: int = 0 + compute_instance_id: int = 0 + memory_total: int = 8192 * _MIB + memory_used: int = 512 * _MIB + ecc_errors: int = 0 + sm_util: float | None = 60.0 + """ + The SM utilization GPM samples for the instance, or None when unreadable. + """ + uuid_readable: bool = True + """ + Whether the driver answers this instance's UUID, i.e. whether it is the one + faulty instance of an otherwise healthy card. + """ + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "T-Head PPU" + bdf: str = "0000:01:00.0" + minor_number: int | None = 3 + compute_capability: tuple[int, int] = (8, 0) + cores: int = 128 + memory_total: int = 65536 * _MIB + memory_used: int = 1024 * _MIB + ecc_errors: int = 0 + cores_utilization: int = 33 + temperature: int = 55 + power_limit: int = 350_000 # mW + power_usage: int = 150_000 # mW + instances: list[_Instance] | None = None + """ + The card's GPU/compute instances, or None for a card with MIG disabled. + """ + + +class _GpmMetrics: + """ + A GPM metrics request, as the detector fills and reads it. + """ + + def __init__(self): + self.version = 0 + self.numMetrics = 0 + self.sample1 = None + self.sample2 = None + self.metrics = [SimpleNamespace(metricId=0, value=float("nan"))] + + +@dataclass +class _FakeHGML: + """ + A stand-in for the pyhgml binding, recording every call the detector makes. + + The error type is the real module's and every HGML_* constant is delegated + to it, so a fake drifting from the binding's contract fails here rather than + on hardware. Entry points are dispatched by name, as the fake pymxsml of + test_metax.py does, which keeps HGML's camelCase out of the handlers' own + names -- pyhgml's surface is wide enough for that to matter. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + gpm_supported: bool = True + + HGMLError = pyhgml.HGMLError + + c_hgmlGpmMetricsGet_t = _GpmMetrics # noqa: N815 + + instance_profile_name = "1g.8gb" + instance_profile_memory_mb = 8192 + instance_profile_cores = 16 + + def __getattr__(self, name: str): + handler = { + "hgmlInit": self._init, + "hgmlSystemGetDriverVersion": self._system_get_driver_version, + "hgmlSystemGetHggcDriverVersion": self._system_get_hggc_driver_version, + "hgmlDeviceGetCount": self._device_get_count, + "hgmlDeviceGetHandleByIndex": self._device_get_handle_by_index, + "hgmlDeviceGetHggcComputeCapability": self._device_get_compute_capability, + "hgmlDeviceGetPciInfo": self._device_get_pci_info, + "hgmlDeviceGetMemoryAffinity": self._device_get_memory_affinity, + "hgmlDeviceGetMinorNumber": self._device_get_minor_number, + "hgmlDeviceGetName": self._device_get_name, + "hgmlDeviceGetUUID": self._device_get_uuid, + "hgmlDeviceGetNumGpuCores": self._device_get_num_gpu_cores, + "hgmlDeviceGetMemoryInfo": self._device_get_memory_info, + "hgmlDeviceGetMemoryErrorCounter": self._device_get_memory_error_counter, + "hgmlDeviceGetPowerManagementDefaultLimit": self._device_get_power_limit, + "hgmlDeviceGetUtilizationRates": self._device_get_utilization_rates, + "hgmlDeviceGetTemperature": self._device_get_temperature, + "hgmlDeviceGetPowerUsage": self._device_get_power_usage, + "hgmlDeviceGetMigMode": self._device_get_mig_mode, + "hgmlDeviceGetMaxMigDeviceCount": self._device_get_max_mig_device_count, + "hgmlDeviceGetMigDeviceHandleByIndex": self._device_get_mig_handle, + "hgmlDeviceGetGpuInstanceId": self._device_get_gpu_instance_id, + "hgmlDeviceGetComputeInstanceId": self._device_get_compute_instance_id, + "hgmlDeviceGetGpuInstanceById": self._device_get_gpu_instance_by_id, + "hgmlGpuInstanceGetComputeInstanceById": self._gi_get_ci_by_id, + "hgmlGpuInstanceGetInfo": self._gi_get_info, + "hgmlComputeInstanceGetInfo": self._ci_get_info, + "hgmlDeviceGetGpuInstanceProfileInfo": self._device_get_gi_profile_info, + "hgmlGpuInstanceGetComputeInstanceProfileInfo": self._gi_get_ci_profile_info, + "hgmlGpmQueryDeviceSupport": self._gpm_query_device_support, + "hgmlGpmSampleAlloc": self._gpm_sample_alloc, + "hgmlGpmSampleFree": self._gpm_sample_free, + "hgmlGpmSampleGet": self._gpm_sample_get, + "hgmlGpmMigSampleGet": self._gpm_mig_sample_get, + "hgmlGpmMetricsGet": self._gpm_metrics_get, + }.get(name) + if handler is None: + if name.startswith("HGML_"): + # Enumeration constants are the real module's, so a fake + # drifting from the binding's contract fails here rather than + # on hardware. + return getattr(pyhgml, name) + msg = f"module pyhgml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args, **kwargs): + self.calls.append(name) + return handler(*args, **kwargs) + + return entry_point + + # System. + + def _init(self) -> None: + pass + + def _system_get_driver_version(self) -> str: + return "1.2.3" + + def _system_get_hggc_driver_version(self) -> int: + return 12030 # i.e. 12.3.0 + + def _device_get_count(self) -> int: + return len(self.cards) + + def _device_get_handle_by_index(self, index: int) -> _Card: + return self.cards[index] + + # Identity and capability. + + def _device_get_compute_capability(self, handle: _Card) -> tuple[int, int]: + return handle.compute_capability + + def _device_get_pci_info(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(busIdLegacy=handle.bdf) + + def _device_get_memory_affinity( + self, + handle: _Card, + node_set_size: int, + scope: int, + ) -> list[int]: + msg = "no memory affinity" + raise pyhgml.HGMLError(msg) + + def _device_get_minor_number(self, handle: _Card) -> int: + if handle.minor_number is None: + msg = "no minor number" + raise pyhgml.HGMLError(msg) + return handle.minor_number + + def _device_get_name(self, handle: _Card) -> str: + return handle.name + + def _device_get_uuid(self, handle: _Card | _Instance) -> str: + if not getattr(handle, "uuid_readable", True): + raise self.HGMLError(pyhgml.HGML_ERROR_NOT_FOUND) + return handle.uuid + + def _device_get_num_gpu_cores(self, handle: _Card) -> int: + return handle.cores + + def _device_get_power_limit(self, handle: _Card) -> int: + return handle.power_limit + + # Memory. + + def _device_get_memory_info(self, handle: _Card | _Instance) -> SimpleNamespace: + return SimpleNamespace( + total=handle.memory_total, + used=handle.memory_used, + ) + + def _device_get_memory_error_counter( + self, + handle: _Card | _Instance, + error_type: int, + counter_type: int, + location_type: int, + ) -> int: + return handle.ecc_errors + + # Usage. + + def _device_get_utilization_rates(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(gpu=handle.cores_utilization, memory=0) + + def _device_get_temperature(self, handle: _Card, sensor: int) -> int: + return handle.temperature + + def _device_get_power_usage(self, handle: _Card) -> int: + return handle.power_usage + + def _gpm_query_device_support(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(isSupportedDevice=int(self.gpm_supported)) + + def _gpm_sample_alloc(self) -> object: + return object() + + def _gpm_sample_free(self, sample: object) -> None: + pass + + def _gpm_sample_get(self, handle: _Card, sample: object) -> None: + self._gpm_target = handle + + def _gpm_mig_sample_get( + self, + handle: _Card, + gpu_instance_id: int, + sample: object, + ) -> None: + self._gpm_target = next( + ( + inst + for inst in handle.instances or [] + if inst.gpu_instance_id == gpu_instance_id + ), + None, + ) + + def _gpm_metrics_get(self, metrics_get: _GpmMetrics) -> None: + sm_util = getattr(self._gpm_target, "sm_util", None) + metrics_get.metrics[0].value = ( + float("nan") if sm_util is None else float(sm_util) + ) + + # GPU/compute instances. + + def _device_get_mig_mode(self, handle: _Card) -> tuple[int, int]: + mode = ( + pyhgml.HGML_DEVICE_MIG_DISABLE + if handle.instances is None + else pyhgml.HGML_DEVICE_MIG_ENABLE + ) + return mode, mode + + def _device_get_max_mig_device_count(self, handle: _Card) -> int: + return 8 + + def _device_get_mig_handle(self, handle: _Card, index: int) -> _Instance: + instances = handle.instances or [] + if index >= len(instances): + msg = "no instance at that slot" + raise pyhgml.HGMLError(msg) + return instances[index] + + def _device_get_gpu_instance_id(self, handle: _Instance) -> int: + return handle.gpu_instance_id + + def _device_get_compute_instance_id(self, handle: _Instance) -> int: + return handle.compute_instance_id + + def _device_get_gpu_instance_by_id( + self, + handle: _Card, + gpu_instance_id: int, + ) -> SimpleNamespace: + return SimpleNamespace(card=handle, gpu_instance_id=gpu_instance_id) + + def _gi_get_ci_by_id( + self, + gpu_instance: SimpleNamespace, + compute_instance_id: int, + ) -> SimpleNamespace: + return SimpleNamespace(compute_instance_id=compute_instance_id) + + def _gi_get_info(self, gpu_instance: SimpleNamespace) -> SimpleNamespace: + return SimpleNamespace(profileId=0) + + def _ci_get_info(self, compute_instance: SimpleNamespace) -> SimpleNamespace: + return SimpleNamespace(profileId=0) + + def _device_get_gi_profile_info( + self, + handle: _Card, + profile_id: int, + ) -> SimpleNamespace: + # Only the 1-slice profile exists, i.e. HGML_GPU_INSTANCE_PROFILE_1_SLICE. + if profile_id != pyhgml.HGML_GPU_INSTANCE_PROFILE_1_SLICE: + msg = "no such GPU instance profile" + raise pyhgml.HGMLError(msg) + return SimpleNamespace( + id=0, + memorySizeMB=self.instance_profile_memory_mb, + name=self.instance_profile_name, + ) + + def _gi_get_ci_profile_info( + self, + gpu_instance: SimpleNamespace, + profile_id: int, + engine_profile_id: int, + ) -> SimpleNamespace: + if (profile_id, engine_profile_id) != ( + pyhgml.HGML_COMPUTE_INSTANCE_PROFILE_1_SLICE, + pyhgml.HGML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED, + ): + msg = "no such compute instance profile" + raise pyhgml.HGMLError(msg) + return SimpleNamespace(id=0, multiprocessorCount=self.instance_profile_cores) + + +_USAGE_CALLS = ( + "hgmlGpmQueryDeviceSupport", + "hgmlGpmSampleGet", + "hgmlGpmMigSampleGet", + "hgmlGpmMetricsGet", + "hgmlDeviceGetUtilizationRates", + "hgmlDeviceGetTemperature", + "hgmlDeviceGetPowerUsage", +) +""" +The driver calls only the usage query is allowed to make. Deliberately not the +whole metric-looking surface: hgmlDeviceGetPowerManagementDefaultLimit reads the +power *limit* and hgmlDeviceGetMemoryInfo the memory *total*, both inventory. +""" + +_CARD_MEMORY_UTILIZATION = 1.56 +""" +A default card's memory utilization: 1024 MiB used of 65536 MiB total, as +get_utilization rounds it. +""" + +_INSTANCE_MEMORY_UTILIZATION = 6.25 +""" +A default instance's memory utilization: 512 MiB used of 8192 MiB total. +""" + + +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the ECC error check on: it is opt-in, as reading the counters costs a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults + to true. A real module attribute is set because the env lookup is cached. + """ + monkeypatch.setattr( + thead.envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + +@pytest.fixture +def detector(monkeypatch): + """ + Build a T-Head detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card, **kwargs) -> tuple[THeadDetector, _FakeHGML]: + fake = _FakeHGML(cards=list(cards), **kwargs) + monkeypatch.setattr(thead, "pyhgml", fake) + # is_supported() initializes the real driver, which the fake replaces. + monkeypatch.setattr(THeadDetector, "is_supported", staticmethod(lambda: True)) + # The NUMA node comes from sysfs, so it is answered here instead of + # letting the host decide what the test sees. + monkeypatch.setattr(thead, "get_numa_node_by_bdf", lambda *_: "") + # GPM samples over a 100 ms window of real time, twice per query. + monkeypatch.setattr(thead.time, "sleep", lambda *_: None) + return THeadDetector(), fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: inventory only, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.THEAD + assert dev.index == 0 + assert dev.name == "T-Head PPU" + assert dev.uuid == "PPU-0" + assert dev.driver_version == "1.2.3" + assert dev.runtime_version == "12.3" + assert dev.runtime_version_original == "12.3.0" + assert dev.compute_capability == "8.0" + assert dev.cores == 128 + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["mig"] is False + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector( + _Card(uuid="PPU-0"), + _Card(uuid="PPU-1", instances=[_Instance(uuid="PPU-1-MIG-0")]), + ) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # The inventory reads that only look like metrics stay. + assert "hgmlDeviceGetPowerManagementDefaultLimit" in fake.calls + assert "hgmlDeviceGetMemoryInfo" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card(uuid="PPU-1", instances=[_Instance(uuid="PPU-1-MIG-0")]), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + for inst in dev.appendix.get("mig_devices", []): + assert "vgpu" not in inst["appendix"] + + +# --------------------------------------------------------------------------- # +# The GPU/compute instance entries, which the operator has no equivalent of. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_keeps_the_instance_entries(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card( + uuid="PPU-1", + instances=[ + _Instance(uuid="PPU-1-MIG-0"), + _Instance(uuid="PPU-1-MIG-1", gpu_instance_id=1), + ], + ), + ) + + devices = det.detect_info() + + # The physical card stays the reported device, whether or not MIG is on. + assert [dev.uuid for dev in devices] == ["PPU-0", "PPU-1"] + assert "mig_devices" not in devices[0].appendix + + instances = devices[1].appendix["mig_devices"] + assert [inst["uuid"] for inst in instances] == ["PPU-1-MIG-0", "PPU-1-MIG-1"] + # `sliced` is what the topology path keys off, so it must survive. + assert all(inst["appendix"]["sliced"] is True for inst in instances) + assert all(inst["appendix"]["mig"] is True for inst in instances) + assert all(inst["appendix"]["bdf"] == "0000:01:00.0" for inst in instances) + assert [inst["appendix"]["gpu_instance_id"] for inst in instances] == [0, 1] + assert [inst["name"] for inst in instances] == ["1g.8gb"] * 2 + assert [inst["cores"] for inst in instances] == [16] * 2 + assert [inst["memory"] for inst in instances] == [8192] * 2 + # index_mig_devices numbers them above the cards, one block per card. + assert [inst["index"] for inst in instances] == [2 + 1 * 8, 2 + 1 * 8 + 1] + # The instances' usage fields keep a Device's defaults as well. + assert [inst["cores_utilization"] for inst in instances] == [0] * 2 + assert [inst["memory_used"] for inst in instances] == [0] * 2 + assert [inst["temperature"] for inst in instances] == [None] * 2 + assert [inst["power_used"] for inst in instances] == [None] * 2 + + +# --------------------------------------------------------------------------- # +# The minor number, recorded only when the driver answers. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_records_the_minor_number(detector): + det, _ = detector(_Card(uuid="PPU-0", minor_number=3)) + + dev = det.detect_info()[0] + + assert dev.appendix["minor_number"] == 3 + # Device.index stays the enumeration index. + assert dev.index == 0 + + +def test_detect_info_omits_the_minor_number_when_unreadable(detector): + det, _ = detector(_Card(uuid="PPU-0", minor_number=None)) + + dev = det.detect_info()[0] + + # Absent rather than substituted by the enumeration index, as the operator + # does: a substituted value would make a wrong ordinal look proven. + assert "minor_number" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID, instances included. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card( + uuid="PPU-1", + cores_utilization=7, + memory_used=2048 * _MIB, + temperature=61, + power_usage=200_000, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["PPU-0"].cores_utilization == 33 + assert by_uuid["PPU-0"].memory_used == 1024 + assert by_uuid["PPU-0"].memory_utilization == _CARD_MEMORY_UTILIZATION + assert by_uuid["PPU-0"].temperature == 55 + assert by_uuid["PPU-0"].power_used == 150 + assert by_uuid["PPU-1"].cores_utilization == 7 + assert by_uuid["PPU-1"].memory_used == 2048 + assert by_uuid["PPU-1"].temperature == 61 + assert by_uuid["PPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["PPU-0"].index == 0 + assert by_uuid["PPU-0"].memory == 65536 + assert by_uuid["PPU-0"].power == 350 + assert by_uuid["PPU-0"].name == "T-Head PPU" + + +def test_detect_usage_merges_the_instance_entries(detector): + det, _ = detector( + _Card( + uuid="PPU-0", + instances=[ + _Instance(uuid="PPU-0-MIG-0", sm_util=42.0), + _Instance(uuid="PPU-0-MIG-1", gpu_instance_id=1, sm_util=None), + ], + ), + ) + + devices = det.detect() + + instances = devices[0].appendix["mig_devices"] + assert [inst["cores_utilization"] for inst in instances] == [42, None] + assert [inst["memory_used"] for inst in instances] == [512] * 2 + assert [inst["memory_utilization"] for inst in instances] == [ + _INSTANCE_MEMORY_UTILIZATION, + ] * 2 + # An instance reports neither temperature nor power, so it carries the card's. + assert [inst["temperature"] for inst in instances] == [55] * 2 + assert [inst["power_used"] for inst in instances] == [150] * 2 + # The inventory fields survive the merge untouched. + assert [inst["name"] for inst in instances] == ["1g.8gb"] * 2 + assert [inst["memory"] for inst in instances] == [8192] * 2 + assert all(inst["appendix"]["sliced"] is True for inst in instances) + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["PPU-0"] + assert devices[0].name == "T-Head PPU" + assert devices[0].memory == 65536 + assert devices[0].cores_utilization == 33 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + dev = det.detect()[0] + + assert dev.name == "T-Head PPU" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.cores_utilization == 33 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _CARD_MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.usefixtures("health_check") +def test_detect_usage_bounds_a_faulty_instance_to_itself(detector): + # One instance refusing its UUID used to abort the whole card's MIG loop, so + # every later instance kept the inventory's defaults -- 0 % and 0 MiB, i.e. + # reported idle while it may be running a workload. + before_util, after_util = 42.0, 17.0 + det, _ = detector( + _Card( + uuid="PPU-0", + instances=[ + _Instance(uuid="PPU-0-MIG-0", sm_util=before_util), + _Instance(uuid="PPU-0-MIG-1", gpu_instance_id=1, uuid_readable=False), + _Instance(uuid="PPU-0-MIG-2", gpu_instance_id=2, sm_util=after_util), + ], + ), + ) + + mig_devs = det.detect()[0].appendix["mig_devices"] + + by_uuid = {m["uuid"]: m for m in mig_devs} + assert by_uuid["PPU-0-MIG-0"]["cores_utilization"] == before_util + # The instance past the faulty one is still enumerated, with its own reading. + assert by_uuid["PPU-0-MIG-2"]["cores_utilization"] == after_util + + +@pytest.mark.usefixtures("health_check") +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector( + _Card(uuid="PPU-0", instances=[_Instance(uuid="PPU-0-MIG-0")]), + ) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. The health + # check has to be switched on for this to pin anything: with it off both + # queries answer HEALTHY without a driver call, and the assertion would hold + # whether or not the usage query re-read health at all. + dev = det.detect()[0] + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert ( + dev.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.HEALTHY + ) + + +@pytest.mark.usefixtures("health_check") +def test_detect_reports_an_uncorrectable_ecc_error(detector): + det, _ = detector( + _Card( + uuid="PPU-0", + ecc_errors=3, + instances=[_Instance(uuid="PPU-0-MIG-0", ecc_errors=1)], + ), + ) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + info = det.detect_info()[0] + assert info.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert ( + info.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.UNHEALTHY + ) + dev = det.detect()[0] + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert ( + dev.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.UNHEALTHY + ) + + +# --------------------------------------------------------------------------- # +# CDI: /dev/alixpu_ppu{N} is named after the enumeration index. # +# --------------------------------------------------------------------------- # + + +def _fake_node_factory(seen_paths: list[str], minors: dict[str, int]): + """ + Stand in for device_to_cdi_device_node, giving each node a kernel minor. + + The minor is what proves an ordinal-named node reaches the card whose record + names that minor, so a fake that cannot carry one cannot exercise the check. + """ + + def _fake_device_node(path): + seen_paths.append(path) + return SimpleNamespace(path=path, minor=minors.get(path, 0)) + + return _fake_device_node + + +def test_cdi_device_node_path_is_named_after_the_enumeration_index(monkeypatch): + seen_paths: list[str] = [] + + monkeypatch.setattr( + cdi_thead, + "device_to_cdi_device_node", + # The node the ordinal names carries the minor the record states, which + # is the host this fixture reproduces. + _fake_node_factory(seen_paths, {"/dev/alixpu_ppu0": 7}), + ) + + devices = [ + Device( + manufacturer=ManufacturerEnum.THEAD, + index=0, + name="T-Head PPU", + uuid="PPU-0", + memory=65536, + appendix={"bdf": "0000:01:00.0", "minor_number": 7}, + ), + Device( + manufacturer=ManufacturerEnum.THEAD, + index=1, + name="T-Head PPU", + uuid="PPU-1", + memory=65536, + appendix={"bdf": "0000:02:00.0"}, + ), + ] + + config = THeadGenerator().generate(devices) + + assert config is not None + # Unlike /dev/nvidia{N} and /dev/iluvatar{N}, the T-Head node is named after + # the card ordinal and not after the driver's minor number, which the + # operator records purely to PROVE a node addresses the card it describes. + # So the appendix number does not name the node: a card carrying one is + # addressed by its enumeration index all the same, as is one carrying none. + assert "/dev/alixpu_ppu0" in seen_paths + assert "/dev/alixpu_ppu1" in seen_paths + assert "/dev/alixpu_ppu7" not in seen_paths + + +def test_cdi_refuses_a_node_carrying_another_cards_minor(monkeypatch): + # The ordinal names the node and the recorded minor proves that name reaches + # the card the record describes. On the operator's measured host a path built + # from the record lands on the NEIGHBOURING accelerator silently, which is + # the failure this refuses: card 0's record says minor 7, but the node its + # ordinal names carries 8, so that ordinal is addressing another card. + seen_paths: list[str] = [] + monkeypatch.setattr( + cdi_thead, + "device_to_cdi_device_node", + _fake_node_factory(seen_paths, {"/dev/alixpu_ppu0": 8}), + ) + + devices = [ + Device( + manufacturer=ManufacturerEnum.THEAD, + index=0, + name="T-Head PPU", + uuid="PPU-0", + memory=65536, + appendix={"bdf": "0000:01:00.0", "minor_number": 7}, + ), + ] + + config = THeadGenerator().generate(devices) + + # The path was built and then rejected, so no device survives to be handed + # over -- a missing accelerator, never someone else's. + assert "/dev/alixpu_ppu0" in seen_paths + assert config is None From 265a0437a55cc3cf2174f3cac34617f47d6f0d30 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:52:30 +0800 Subject: [PATCH 11/12] feat(deployer): pin the device ordering, and report why a container is not running - add `Deployer.map_visible_devices_ordering`, resolving the runtime visible devices env names to `CUDA_DEVICE_ORDER=PCI_BUS_ID` when NVIDIA is among them, and inject it from all three deployers when a container requests all devices or runs privileged -- the deployers' own resolution of "this container sees every card" -- as a default the container's own declaration overrides - add `WorkloadStatusExit`, holding a container's name and operation token, its exit code, reason, message, timestamps and restart count, held on `WorkloadStatus.exits` and defaulted so a payload serialized before this change still deserializes. The exit code stays optional: a container blocked from starting reports a reason and no code. - add one `parse_container_exit` beside it rather than a copy per runtime: podman-py is Docker-API compatible, both call sites inspect a reloaded container, and the two parsers were 58 of 60 lines identical, so every fix would have had to land twice and could drift - derive reason "Error" from a non-zero exit code, since `State.Error` is filled for a container that failed to start and not for one that ran and exited non-zero -- so the commonest crash of all produced an exit code with no reason, and the caller only sets `state_message` when there is one. Kubernetes already reported "Error" for the same event. - report an unset timestamp as empty rather than Go's zero time, which a UI renders as year 1, and truncate Docker's 9-digit nanoseconds to the 6 the Kubernetes deployer formats, which is also all strptime's %f accepts - read every `State` key defensively, so a container missing them degrades to the code alone, and cover a "restarting" container: a crash-looping one sits there with the last attempt's exit code - leave `parse_state`'s verdicts alone on Docker and Podman: this adds information, it does not re-decide the state - report Failed instead of Pending for ErrImageNeverPull, ErrImagePull, ImagePullBackOff, InvalidImageName and RegistryUnavailable, so a model that can never start stops looking like one that is still starting, and build an exit entry per Kubernetes container from its terminated or waiting state, init containers included - append the Pod's Warning event, which carries the registry error the bare waiting reason omits, reading events only for a Pod blocked that way, selecting by `involvedObject.uid` as well as name so a recreated Pod does not match its predecessor's, sorting by timestamp because the API guarantees no order and an Event's random name suffix breaks list order, and preferring the kubelet's "Failed" Event so a stale FailedScheduling cannot stand in as the diagnosis - degrade a failed events read to a debug log, so a cluster that has not applied the new RBAC rule keeps reporting the state and the reason, and grant get/list on core events as its own rule rather than widening the wildcard one Signed-off-by: thxCode --- deploy/manifests/kubernetes.yaml | 11 + gpustack_runtime/deployer/__types__.py | 206 +++++++- gpustack_runtime/deployer/docker.py | 20 + gpustack_runtime/deployer/kuberentes.py | 255 +++++++++- gpustack_runtime/deployer/podman.py | 20 + .../deployer/test_docker_status.py | 181 +++++++ .../deployer/test_kubernetes_status.py | 446 ++++++++++++++++++ .../deployer/test_podman_status.py | 144 ++++++ .../deployer/test_visible_devices_ordering.py | 349 ++++++++++++++ .../deployer/test_workload_status.py | 77 ++- 10 files changed, 1706 insertions(+), 3 deletions(-) create mode 100644 tests/gpustack_runtime/deployer/test_docker_status.py create mode 100644 tests/gpustack_runtime/deployer/test_kubernetes_status.py create mode 100644 tests/gpustack_runtime/deployer/test_podman_status.py create mode 100644 tests/gpustack_runtime/deployer/test_visible_devices_ordering.py diff --git a/deploy/manifests/kubernetes.yaml b/deploy/manifests/kubernetes.yaml index ec9b76d..c00de7f 100644 --- a/deploy/manifests/kubernetes.yaml +++ b/deploy/manifests/kubernetes.yaml @@ -25,6 +25,17 @@ rules: - "pods/exec" verbs: - "*" + # Allow reading the Events of a workload Pod, so an image-pull failure + # reports the registry error behind a bare "ImagePullBackOff" reason. + # Kept as its own rule with only "get"/"list": the rule above grants "*", + # which would also allow creating and deleting Events. + - apiGroups: + - "" + resources: + - "events" + verbs: + - "get" + - "list" # Allow resolving the vendor RuntimeClass (e.g. "ascend", "nvidia") for # accelerated workload Pods, see GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_CLASS. - apiGroups: diff --git a/gpustack_runtime/deployer/__types__.py b/gpustack_runtime/deployer/__types__.py index c385207..6f6af93 100644 --- a/gpustack_runtime/deployer/__types__.py +++ b/gpustack_runtime/deployer/__types__.py @@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from dataclasses_json import dataclass_json @@ -1130,6 +1130,165 @@ def name_rfc1123_guard(self) -> WorkloadName: return self._name_rfc1123_guard +@dataclass_json +@dataclass +class WorkloadStatusExit: + """ + An exit information of a workload's container. + + Attributes: + name (str): + Name representing the exited target, e.g., human-readable container name. + token (WorkloadOperationToken): + Token of the exited target, e.g., container ID. + exit_code (int | None): + Exit code of the container. + reason (str): + Short reason of the termination or the blocking. + message (str): + Detailed message of the termination or the blocking. + started_at (str): + Start time of the container's last execution. + finished_at (str): + Finish time of the container's last execution. + restart_count (int): + Restart times of the container. + + """ + + name: str + """ + Name representing the exited target, e.g., human-readable container name. + """ + token: WorkloadOperationToken + """ + Token of the exited target, e.g. container ID, + which addresses the same container as the logs/exec operations. + """ + exit_code: int | None = None + """ + Exit code of the container, + None if the container has not terminated, e.g. blocked from starting. + """ + reason: str = "" + """ + Short reason of the termination or the blocking, + e.g. "OOMKilled", "Error", "ImagePullBackOff". + """ + message: str = "" + """ + Detailed message of the termination or the blocking. + """ + started_at: str = "" + """ + Start time of the container's last execution, + empty if the container has never started. + """ + finished_at: str = "" + """ + Finish time of the container's last execution, + empty if the container has not finished. + """ + restart_count: int = 0 + """ + Restart times of the container. + """ + + +_CONTAINER_EXITED_STATUSES = ("exited", "dead", "restarting") +""" +Container statuses carrying an exit code. "restarting" is one of them: a +crash-looping container sits there between attempts with the last attempt's +ExitCode already filled in, and that code is the diagnosis being asked for. +""" + +_CONTAINER_UNSET_TIMESTAMP_PREFIX = "0001-01-01" +""" +What Docker fills an unset timestamp with -- Go's zero time -- rather than +omitting the key, which is why a plain `.get(..., "")` never defaults. +""" + + +def parse_container_exit( + c: Any, + component_name_label: str, +) -> WorkloadStatusExit | None: + """ + Build the exit entry of a container that has terminated. + + Docker and Podman report the same State object -- podman-py is Docker-API + compatible and both call sites inspect a fully reloaded container -- so both + deployers share this. Keeping it in one place is the point: the reason + derivation and the timestamp normalization below are exactly the kind of + detail that silently drifts when it lives in two files. + + Args: + c: + A Docker-API-compatible container to inspect. + component_name_label: + The label the deployer names its components with. + + Returns: + A WorkloadStatusExit if the container has terminated, None if it is + still running or pending and so contributes nothing. + + """ + if c.status not in _CONTAINER_EXITED_STATUSES: + return None + + # Every read is .get()-able: a container whose State is missing keys (or + # missing entirely) must degrade gracefully, never raise. + state = c.attrs.get("State", {}) or {} + + exit_code = state.get("ExitCode") + if state.get("OOMKilled"): + reason = "OOMKilled" + elif state.get("Error"): + reason = "Error" + elif exit_code not in (0, None): + # Docker fills State.Error for a container that failed to *start*, not + # for one that ran and exited non-zero -- the commonest crash of all. + # Without this the caller has an exit code and no reason, so it sets no + # state_message, while the Kubernetes deployer reports "Error" for the + # same event. Same field, same answer, either backend. + reason = "Error" + else: + reason = "" + + return WorkloadStatusExit( + name=c.labels.get(component_name_label, "") or c.name, + token=c.attrs.get("Id", "") or c.name, + exit_code=exit_code, + reason=reason, + message=state.get("Error", ""), + started_at=_parse_container_timestamp(state.get("StartedAt")), + finished_at=_parse_container_timestamp(state.get("FinishedAt")), + restart_count=c.attrs.get("RestartCount", 0), + ) + + +def _parse_container_timestamp(value: str | None) -> str: + """ + Normalize a container's State timestamp to what WorkloadStatusExit promises. + + Args: + value: + The raw timestamp as the container's State reports it. + + Returns: + The timestamp, empty if it never happened. + + """ + if not value or value.startswith(_CONTAINER_UNSET_TIMESTAMP_PREFIX): + # WorkloadStatusExit documents these as empty when they never happened, + # and Go's zero time rendered by a UI reads as "January 1, year 1". + return "" + # Docker reports 9-digit nanoseconds where the Kubernetes deployer formats 6 + # (_TIMESTAMP_FORMAT), and strptime's %f accepts at most 6, so a consumer + # parsing this field would succeed on one backend and raise on the other. + return re.sub(r"(\.\d{6})\d+", r"\1", value) + + @dataclass_json @dataclass class WorkloadStatus: @@ -1153,6 +1312,9 @@ class WorkloadStatus: The operation for the executable containers of the workload. loggable (list[WorkloadStatusOperation]): The operation for the loggable containers of the workload. + exits (list[WorkloadStatusExit]): + The exit information for the terminated or start-blocked containers + of the workload. state (WorkloadStatusStateEnum): Current state of the workload. @@ -1192,6 +1354,12 @@ class WorkloadStatus: """ The operation for the loggable containers of the workload. """ + exits: list[WorkloadStatusExit] | None = field(default_factory=list) + """ + The exit information for the terminated or start-blocked containers + of the workload, one entry per container. + Containers in neither state are not listed. + """ state: WorkloadStatusStateEnum = WorkloadStatusStateEnum.UNKNOWN """ The current state of the workload. @@ -1619,6 +1787,42 @@ def map_backend_visible_devices( ) return ret + def map_visible_devices_ordering( + self, + runtime_envs: list[str], + ) -> dict[str, str]: + """ + Return the device ordering environment variables + for the given runtime visible devices env names. + + Only meaningful for a container seeing every device of the host: + it must number the devices as the detector, the driver and the vendor + tooling do, otherwise an index computed from detection addresses + another device inside the container. + For example, CUDA's default ordering, CUDA_DEVICE_ORDER=FASTEST_FIRST, + sorts the visible devices by a performance heuristic, which reshuffles + the ordinals on a heterogeneous host, + while PCI_BUS_ID sorts by PCI bus id, as NVML and the detector enumerate, + see https://github.com/gpustack/gpustack/issues/6041. + No other manufacturer documents such an ordering switch, + so none of them contributes a variable. + + Args: + runtime_envs: + The runtime visible devices environment variable names. + + Returns: + A dictionary mapping device ordering environment variable names + to corresponding values. + + """ + if any( + self.get_manufacturer(runtime_env) == ManufacturerEnum.NVIDIA + for runtime_env in runtime_envs + ): + return {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"} + return {} + def map_visible_devices_affinities( self, runtime_envs: list[str], diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index 46a4a71..96ed7db 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -45,6 +45,7 @@ WorkloadStatus, WorkloadStatusOperation, WorkloadStatusStateEnum, + parse_container_exit, ) from .__utils__ import ( _MiB, @@ -279,6 +280,12 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = parse_container_exit(c, _LABEL_COMPONENT_NAME) + if exit_ is not None: + self.exits.append(exit_) + if exit_.reason and not self.state_message: + self.state_message = exit_.message or exit_.reason + self.state = self.parse_state(d_containers) @@ -1111,6 +1118,19 @@ def _create_containers( ) create_options["environment"].update(b_vs) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + if r_v == "all" or privileged: + o_vs = self.map_visible_devices_ordering(runtime_envs) + # Take the ordering as default, + # never overwrite the one declared by the container. + create_options["environment"] = { + **o_vs, + **create_options["environment"], + } + # Configure affinity if applicable. create_options.update( self.map_visible_devices_affinities( diff --git a/gpustack_runtime/deployer/kuberentes.py b/gpustack_runtime/deployer/kuberentes.py index 75ce12a..6437a48 100644 --- a/gpustack_runtime/deployer/kuberentes.py +++ b/gpustack_runtime/deployer/kuberentes.py @@ -5,6 +5,7 @@ import logging import os from dataclasses import dataclass, field +from datetime import datetime, timezone from enum import Enum from pathlib import Path from typing import TYPE_CHECKING @@ -31,6 +32,7 @@ WorkloadOperationToken, WorkloadPlan, WorkloadStatus, + WorkloadStatusExit, WorkloadStatusOperation, WorkloadStatusStateEnum, ) @@ -53,6 +55,29 @@ _LABEL_WORKLOAD = f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/workload" _LABEL_COMPONENT = f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/component" +_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +_IMAGE_PULL_BLOCKED_REASONS = frozenset( + { + "ErrImageNeverPull", + "ErrImagePull", + "ImagePullBackOff", + "InvalidImageName", + "RegistryUnavailable", + }, +) +""" +Container waiting reasons meaning the image can never be pulled as requested, +so the workload has failed instead of merely waiting to start. +""" + +_IMAGE_PULL_FAILED_EVENT_REASON = "Failed" +""" +The reason a kubelet stamps on the Event carrying the registry error, e.g. +"Failed to pull image ...: unauthorized". Any other warning the Pod collected, +a stale FailedScheduling among them, explains something else. +""" + class KubernetesWorkloadServiceTypeEnum(str, Enum): """ @@ -196,6 +221,11 @@ def parse_state( """ match k_pod.status.phase: case "Pending": + # A Pod blocked on an image it can never pull is not waiting for + # anything: report it as failed so the caller stops watching a + # workload that will never become ready. + if KubernetesWorkloadStatus.parse_image_pull_block(k_pod): + return WorkloadStatusStateEnum.FAILED return WorkloadStatusStateEnum.PENDING case "Succeeded": return WorkloadStatusStateEnum.INACTIVE @@ -215,10 +245,171 @@ def parse_state( return WorkloadStatusStateEnum.RUNNING + @staticmethod + def parse_image_pull_block( + k_pod: kubernetes.client.V1Pod, + ) -> kubernetes.client.V1ContainerStateWaiting | None: + """ + Find the container state blocking the workload on an image pull. + + Args: + k_pod: + Pod object from Kubernetes API. + + Returns: + The waiting state of the first container blocked on its image pull, + None if no container is. + + """ + for cs in [ + *(k_pod.status.init_container_statuses or []), + *(k_pod.status.container_statuses or []), + ]: + waiting = cs.state.waiting if cs.state else None + if waiting and waiting.reason in _IMAGE_PULL_BLOCKED_REASONS: + return waiting + return None + + @staticmethod + def _parse_exit( + cs: kubernetes.client.V1ContainerStatus | None, + name: str, + ) -> WorkloadStatusExit | None: + """ + Build the exit entry for a container that has terminated or is blocked + from starting. + + Args: + cs: + Status of the container, None if the Pod reports none yet. + name: + Human-readable name of the container. + + Returns: + A WorkloadStatusExit if the container has terminated or is blocked, + None if it is running or has no state at all, and so contributes + nothing. + + """ + if not cs or not cs.state: + return None + + # Prefer the current termination, and fall back to the previous one, so + # a container that has already restarted still reports why it died. + terminated = cs.state.terminated or ( + cs.last_state.terminated if cs.last_state else None + ) + if terminated: + return WorkloadStatusExit( + name=name, + token=cs.name, + exit_code=terminated.exit_code, + reason=terminated.reason or "", + message=terminated.message or "", + started_at=( + terminated.started_at.strftime(_TIMESTAMP_FORMAT) + if terminated.started_at + else "" + ), + finished_at=( + terminated.finished_at.strftime(_TIMESTAMP_FORMAT) + if terminated.finished_at + else "" + ), + restart_count=cs.restart_count or 0, + ) + + if cs.state.waiting: + # A container blocked from starting has never terminated, + # so it carries a reason but no exit code. + return WorkloadStatusExit( + name=name, + token=cs.name, + reason=cs.state.waiting.reason or "", + message=cs.state.waiting.message or "", + restart_count=cs.restart_count or 0, + ) + + return None + + @staticmethod + def _parse_pod_event_message( + k_pod: kubernetes.client.V1Pod, + core_api: kubernetes.client.CoreV1Api, + ) -> str: + """ + Read the Events of the given Pod and return the most relevant message. + + Args: + k_pod: + Pod object from Kubernetes API. + core_api: + Core API client to read the Events with. + + Returns: + The message of the Pod's latest warning Event, + empty if there is none or if the Events cannot be read. + + """ + field_selector = f"involvedObject.name={k_pod.metadata.name}" + if k_pod.metadata.uid: + # A recreated Pod takes the same name, so selecting by name alone + # also selects its predecessor's Events. + field_selector += f",involvedObject.uid={k_pod.metadata.uid}" + + try: + k_events = core_api.list_namespaced_event( + namespace=k_pod.metadata.namespace, + field_selector=field_selector, + ) + except kubernetes.client.exceptions.ApiException: + # Any API failure degrades to no message rather than propagating: + # this read only enriches a diagnosis the state and the reason + # already carry. The expected one is a 403 on a cluster whose + # manifest predates the Events rule, but a 500 or a timeout is no + # reason to fail a status poll either. + debug_log_exception( + logger, + "Failed to list events of pod %s/%s", + k_pod.metadata.namespace, + k_pod.metadata.name, + ) + return "" + + # The warning Events carry the registry error, e.g. the "Failed" Event + # behind a bare "ImagePullBackOff" reason. + k_warnings = [ + k_event + for k_event in k_events.items or [] + if k_event.type == "Warning" and k_event.message + ] + if not k_warnings: + return "" + + # The API guarantees no ordering and an Event name carries a random + # suffix, so list order is not time order: sort it. An Event carrying no + # timestamp at all sorts first, i.e. loses to any that does. + k_warnings.sort( + key=lambda e: (e.last_timestamp or e.event_time or e.first_timestamp) + or datetime.min.replace(tzinfo=timezone.utc), + ) + + # Among the warnings, the kubelet's image-pull failure is the one being + # diagnosed, so it wins over a later warning about something else. + return next( + ( + k_event.message + for k_event in reversed(k_warnings) + if k_event.reason == _IMAGE_PULL_FAILED_EVENT_REASON + ), + k_warnings[-1].message, + ) + def __init__( self, name: WorkloadName, k_pod: kubernetes.client.V1Pod, + core_api: kubernetes.client.CoreV1Api | None = None, **kwargs, ): created_at = k_pod.metadata.creation_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -244,6 +435,9 @@ def __init__( self._k_pod = k_pod + k_init_css = {cs.name: cs for cs in k_pod.status.init_container_statuses or []} + k_css = {cs.name: cs for cs in k_pod.status.container_statuses or []} + k_pod_annos = k_pod.metadata.annotations or {} for ci, c in enumerate(k_pod.spec.init_containers or []): cn = k_pod_annos.get(f"{_LABEL_COMPONENT}-init-{ci}-name", c.name) @@ -255,6 +449,10 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = self._parse_exit(k_init_css.get(c.name), cn) + if exit_: + self.exits.append(exit_) + for ci, c in enumerate(k_pod.spec.containers): cn = k_pod_annos.get(f"{_LABEL_COMPONENT}-run-{ci}-name", c.name) op = WorkloadStatusOperation( @@ -264,8 +462,38 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = self._parse_exit(k_css.get(c.name), cn) + if exit_: + self.exits.append(exit_) + self.state = self.parse_state(k_pod) - if k_pod.status.message: + + # Only a Pod failed on an image pull reads Events, which is both the + # verdict's own condition and the traffic guard: any other Pod, running + # or pending, costs no extra API call. + image_pull_block = ( + self.parse_image_pull_block(k_pod) + if self.state == WorkloadStatusStateEnum.FAILED + and k_pod.status.phase == "Pending" + else None + ) + if image_pull_block: + # The waiting reason alone ("ImagePullBackOff") omits the registry + # error, which is the part explaining why the pull fails, so append + # the Pod's Event message to it. This diagnosis is more specific + # than the Pod's own status message, hence it wins. + blocked_message = ": ".join( + p for p in [image_pull_block.reason, image_pull_block.message] if p + ) + event_message = ( + self._parse_pod_event_message(k_pod, core_api) if core_api else "" + ) + self.state_message = ( + f"{blocked_message}; {event_message}" + if event_message + else blocked_message + ) + elif k_pod.status.message: # Surface the Pod's status message, e.g. a device-plugin # admission rejection ("UnexpectedAdmissionError: Allocate # failed ...") on Failed Pods. @@ -1318,6 +1546,29 @@ def _create_pod( ], ) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + # This includes requesting all devices under KDP: + # the device plugin allocates every device of the node, + # hence the container still enumerates all of them. + # Never overwrite the ordering declared by the container. + if r_v == "all" or privileged: + declared_envs = {e.name for e in container.env} + container.env.extend( + [ + kubernetes.client.V1EnvVar( + name=o_k, + value=o_v, + ) + for o_k, o_v in self.map_visible_devices_ordering( + runtime_envs, + ).items() + if o_k not in declared_envs + ], + ) + container.resources = kubernetes.client.V1ResourceRequirements( limits=(resources if resources else None), requests=(resources if resources else None), @@ -1860,6 +2111,7 @@ def _get( return KubernetesWorkloadStatus( name=name, k_pod=k_pod, + core_api=core_api, ) @_supported @@ -2041,6 +2293,7 @@ def _list( KubernetesWorkloadStatus( name=k_pod.metadata.labels[_LABEL_WORKLOAD], k_pod=k_pod, + core_api=core_api, ) for k_pod in k_pods.items or [] if ( diff --git a/gpustack_runtime/deployer/podman.py b/gpustack_runtime/deployer/podman.py index 7302a3c..eefe869 100644 --- a/gpustack_runtime/deployer/podman.py +++ b/gpustack_runtime/deployer/podman.py @@ -48,6 +48,7 @@ WorkloadStatus, WorkloadStatusOperation, WorkloadStatusStateEnum, + parse_container_exit, ) from .__utils__ import ( _MiB, @@ -282,6 +283,12 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = parse_container_exit(c, _LABEL_COMPONENT_NAME) + if exit_ is not None: + self.exits.append(exit_) + if exit_.reason and not self.state_message: + self.state_message = exit_.message or exit_.reason + self.state = self.parse_state(d_containers) @@ -1090,6 +1097,19 @@ def _create_containers( ) create_options["environment"].update(b_vs) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + if r_v == "all" or privileged: + o_vs = self.map_visible_devices_ordering(runtime_envs) + # Take the ordering as default, + # never overwrite the one declared by the container. + create_options["environment"] = { + **o_vs, + **create_options["environment"], + } + # Configure affinity if applicable. create_options.update( self.map_visible_devices_affinities( diff --git a/tests/gpustack_runtime/deployer/test_docker_status.py b/tests/gpustack_runtime/deployer/test_docker_status.py new file mode 100644 index 0000000..9c517bb --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_docker_status.py @@ -0,0 +1,181 @@ +from types import SimpleNamespace + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.docker import DockerWorkloadStatus + + +def _container( + status: str, + component: str = "run", + name: str = "gpustack-test-run-0", + component_name: str = "run-0", + container_id: str = "abc123", + state: dict | None = None, + restart_policy: str = "no", + restart_count: int | None = None, + created: str = "2026-07-29T08:00:00.000000Z", +) -> SimpleNamespace: + attrs = { + "Id": container_id, + "Created": created, + "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, + "State": state if state is not None else {}, + } + if restart_count is not None: + attrs["RestartCount"] = restart_count + + return SimpleNamespace( + name=name, + status=status, + attrs=attrs, + labels={ + "runtime.gpustack.ai/component": component, + "runtime.gpustack.ai/component-name": component_name, + }, + ) + + +def test_docker_workload_status_exit_oomkilled_preserves_state(): + # A container exited 137 with OOMKilled true: the existing state verdict + # (FAILED, no restart policy) is preserved, and an exit entry carries the + # exit code plus the OOMKilled reason. + c = _container( + status="exited", + state={ + "ExitCode": 137, + "OOMKilled": True, + "Error": "", + "StartedAt": "2026-07-29T08:00:00.000000Z", + "FinishedAt": "2026-07-29T08:05:00.000000Z", + }, + restart_count=2, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "run-0" + assert exit_.token == "abc123" # noqa: S105 + assert exit_.exit_code == 137 + assert exit_.reason == "OOMKilled" + assert exit_.started_at == "2026-07-29T08:00:00.000000Z" + assert exit_.finished_at == "2026-07-29T08:05:00.000000Z" + assert exit_.restart_count == 2 + assert status.state_message == "OOMKilled" + + +def test_docker_workload_status_running_container_no_exit_entry(): + # A running container contributes no exit entry. + c = _container(status="running", state={"ExitCode": 0}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits == [] + + +def test_docker_workload_status_exit_missing_state_keys_degrades(): + # A container whose State lacks the keys degrades to an entry with the + # exit code alone rather than raising. + c = _container(status="exited", state={"ExitCode": 1}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + # Docker fills State.Error for a container that failed to start, not for one + # that ran and exited non-zero -- the commonest crash there is. The reason is + # derived from the code so this backend says what Kubernetes says. + assert exit_.reason == "Error" + assert exit_.message == "" + assert exit_.started_at == "" + assert exit_.finished_at == "" + assert exit_.restart_count == 0 + # Which is what gets the crash into the reported message at all. + assert status.state_message == "Error" + + +def test_docker_workload_status_exit_reports_no_reason_for_a_clean_exit(): + c = _container(status="exited", state={"ExitCode": 0}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].exit_code == 0 + assert status.exits[0].reason == "" + assert status.state_message == "" + + +def test_docker_workload_status_exit_ignores_gos_zero_time(): + # Docker fills an unset timestamp with Go's zero time rather than omitting + # the key, and WorkloadStatusExit documents these as empty when they never + # happened -- a UI renders the zero time as "January 1, year 1". + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "" + assert status.exits[0].finished_at == "" + + +def test_docker_workload_status_exit_truncates_nanoseconds(): + # Docker reports 9-digit nanoseconds where the Kubernetes deployer formats + # 6, and strptime's %f takes at most 6, so a consumer parsing this field + # would succeed on one backend and raise on the other. + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "2026-07-29T08:00:00.123456789Z", + "FinishedAt": "2026-07-29T08:05:00.987654321Z", + }, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "2026-07-29T08:00:00.123456Z" + assert status.exits[0].finished_at == "2026-07-29T08:05:00.987654Z" + + +def test_docker_workload_status_exit_covers_a_restarting_container(): + # A crash-looping container sits in "restarting" between attempts with the + # last attempt's exit code already filled in, which is the diagnosis being + # asked for: whether the user gets it must not depend on when the poll lands. + c = _container(status="restarting", state={"ExitCode": 1}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 1 + + +def test_docker_workload_status_exit_includes_init_containers(): + # Init containers get an exit entry too, not just run containers. + init_c = _container( + status="exited", + component="init", + name="gpustack-test-init-0", + component_name="init-0", + container_id="init-abc", + state={"ExitCode": 1, "Error": "context deadline exceeded"}, + ) + run_c = _container(status="running", state={}) + + status = DockerWorkloadStatus(name="test", d_containers=[init_c, run_c]) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "init-0" + assert exit_.token == "init-abc" # noqa: S105 + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "context deadline exceeded" diff --git a/tests/gpustack_runtime/deployer/test_kubernetes_status.py b/tests/gpustack_runtime/deployer/test_kubernetes_status.py new file mode 100644 index 0000000..c46a443 --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_kubernetes_status.py @@ -0,0 +1,446 @@ +# The cases below drive the Kubernetes deployer's own status parsing, which is +# where the exit list, the image-pull verdict and the Pod Events read become +# observable without a live cluster: hand-built kubernetes.client objects plus a +# fake CoreV1Api carrying a call log. + +from datetime import datetime, timezone + +import kubernetes.client +import pytest + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.kuberentes import KubernetesWorkloadStatus + +_REGISTRY_MESSAGE = 'Back-off pulling image "does-not-exist.invalid/x:y"' +_EVENT_MESSAGE = ( + 'Failed to pull image "does-not-exist.invalid/x:y": ' + "failed to resolve reference: unauthorized" +) + + +class _FakeCoreV1Api: + """ + Stand-in for the Kubernetes CoreV1Api. + + Every call is recorded, so a test can assert that an Events read did *not* + happen - which cannot be observed from the parsed status alone. + """ + + def __init__( + self, + events: list[kubernetes.client.CoreV1Event] | None = None, + raises: Exception | None = None, + ): + self.calls: list[tuple[str, dict]] = [] + self._events = events or [] + self._raises = raises + + def list_namespaced_event(self, **kwargs): + self.calls.append(("list_namespaced_event", kwargs)) + if self._raises: + raise self._raises + return kubernetes.client.CoreV1EventList(items=self._events) + + +def _event( + message: str = _EVENT_MESSAGE, + type_: str = "Warning", + reason: str = "Failed", + last_timestamp: datetime | None = None, +) -> kubernetes.client.CoreV1Event: + return kubernetes.client.CoreV1Event( + metadata=kubernetes.client.V1ObjectMeta(name="gpustack-test.1"), + involved_object=kubernetes.client.V1ObjectReference( + kind="Pod", + name="gpustack-test", + namespace="default", + ), + message=message, + reason=reason, + type=type_, + last_timestamp=last_timestamp, + ) + + +def _pod( + phase: str, + container_statuses: list[kubernetes.client.V1ContainerStatus] | None = None, + init_container_statuses: list[kubernetes.client.V1ContainerStatus] | None = None, + init_containers: list[kubernetes.client.V1Container] | None = None, + message: str | None = None, + annotations: dict[str, str] | None = None, + container_names: list[str] | None = None, + uid: str | None = "gpustack-test-uid", +) -> kubernetes.client.V1Pod: + return kubernetes.client.V1Pod( + metadata=kubernetes.client.V1ObjectMeta( + name="gpustack-test", + namespace="default", + uid=uid, + creation_timestamp=datetime(2026, 8, 14, 8, 0, 0, tzinfo=timezone.utc), + labels={"app": "test"}, + annotations=annotations, + ), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container(name=n) + for n in (container_names or ["run-0"]) + ], + init_containers=init_containers, + ), + status=kubernetes.client.V1PodStatus( + phase=phase, + message=message, + container_statuses=container_statuses, + init_container_statuses=init_container_statuses, + ), + ) + + +def _waiting_status( + name: str = "run-0", + reason: str = "ImagePullBackOff", + message: str = _REGISTRY_MESSAGE, + restart_count: int = 0, +) -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="does-not-exist.invalid/x:y", + image_id="", + ready=False, + restart_count=restart_count, + state=kubernetes.client.V1ContainerState( + waiting=kubernetes.client.V1ContainerStateWaiting( + reason=reason, + message=message, + ), + ), + ) + + +def _terminated_status( + name: str = "run-0", + exit_code: int = 1, + reason: str = "Error", + restart_count: int = 2, +) -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=False, + restart_count=restart_count, + state=kubernetes.client.V1ContainerState( + terminated=kubernetes.client.V1ContainerStateTerminated( + exit_code=exit_code, + reason=reason, + message="the process exited", + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + finished_at=datetime(2026, 8, 14, 8, 2, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def _restarted_status( + name: str = "run-0", +) -> kubernetes.client.V1ContainerStatus: + # A container that has been restarted after an OOM kill: the current state + # is running, the termination only survives in last_state. + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=True, + restart_count=1, + state=kubernetes.client.V1ContainerState( + running=kubernetes.client.V1ContainerStateRunning( + started_at=datetime(2026, 8, 14, 8, 3, 0, tzinfo=timezone.utc), + ), + ), + last_state=kubernetes.client.V1ContainerState( + terminated=kubernetes.client.V1ContainerStateTerminated( + exit_code=137, + reason="OOMKilled", + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + finished_at=datetime(2026, 8, 14, 8, 2, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def _ready_status(name: str = "run-0") -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=True, + restart_count=0, + state=kubernetes.client.V1ContainerState( + running=kubernetes.client.V1ContainerStateRunning( + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def test_image_pull_backoff_reports_failed_with_event_message(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + # A Pod stuck on an unpullable image has failed, it is not pending. + assert status.state == WorkloadStatusStateEnum.FAILED + # The waiting reason alone omits the registry error, so the Event message + # is appended to it. + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + assert _EVENT_MESSAGE in status.state_message + + assert len(status.exits) == 1 + assert status.exits[0].name == "run-0" + assert status.exits[0].token == "run-0" # noqa: S105 + assert status.exits[0].exit_code is None + assert status.exits[0].reason == "ImagePullBackOff" + assert status.exits[0].message == _REGISTRY_MESSAGE + + # Exactly one Events read, field-selected to this Pod. The UID is part of the + # selector because a recreated Pod takes the same name. + assert [c[0] for c in api.calls] == ["list_namespaced_event"] + assert api.calls[0][1]["namespace"] == "default" + assert api.calls[0][1]["field_selector"] == ( + "involvedObject.name=gpustack-test,involvedObject.uid=gpustack-test-uid" + ) + + +def test_pod_event_message_takes_the_latest_by_timestamp(): + # The API returns Events in no particular order, so the newest one is not + # necessarily last in the list. + stale = _event( + message="0/3 nodes are available: insufficient nvidia.com/gpu.", + reason="FailedScheduling", + last_timestamp=datetime(2026, 8, 14, 7, 0, 0, tzinfo=timezone.utc), + ) + latest = _event(last_timestamp=datetime(2026, 8, 14, 9, 0, 0, tzinfo=timezone.utc)) + api = _FakeCoreV1Api(events=[latest, stale]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert _EVENT_MESSAGE in status.state_message + assert "FailedScheduling" not in status.state_message + + +def test_pod_event_message_prefers_the_image_pull_failure(): + # A warning about something else, stamped later than the pull failure, must + # not stand in as the diagnosis of an image pull that cannot succeed. + later_unrelated = _event( + message="Liveness probe failed: connection refused", + reason="Unhealthy", + last_timestamp=datetime(2026, 8, 14, 10, 0, 0, tzinfo=timezone.utc), + ) + pull_failure = _event( + last_timestamp=datetime(2026, 8, 14, 9, 0, 0, tzinfo=timezone.utc), + ) + api = _FakeCoreV1Api(events=[pull_failure, later_unrelated]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert _EVENT_MESSAGE in status.state_message + assert "Liveness probe" not in status.state_message + + +@pytest.mark.parametrize( + "reason", + [ + "ErrImageNeverPull", + "ErrImagePull", + "ImagePullBackOff", + "InvalidImageName", + "RegistryUnavailable", + ], +) +def test_image_pull_blocked_reasons_report_failed(reason): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status(reason=reason)]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert reason in status.state_message + + +def test_pending_on_another_reason_stays_pending(): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + container_statuses=[ + _waiting_status(reason="ContainerCreating", message=""), + ], + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.PENDING + # Not a blocked Pod, so no Events traffic. + assert api.calls == [] + + +def test_terminated_container_reports_exit_code(): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Failed", container_statuses=[_terminated_status()]), + core_api=api, + ) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "the process exited" + assert exit_.started_at == "2026-08-14T08:01:00.000000Z" + assert exit_.finished_at == "2026-08-14T08:02:00.000000Z" + assert exit_.restart_count == 2 + # Events are read for blocked Pods only. + assert api.calls == [] + + +def test_restarted_container_reports_last_terminated_state(): + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Running", container_statuses=[_restarted_status()]), + ) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 137 + assert status.exits[0].reason == "OOMKilled" + assert status.exits[0].restart_count == 1 + + +def test_events_forbidden_degrades_to_state_and_reason(): + api = _FakeCoreV1Api( + raises=kubernetes.client.exceptions.ApiException(status=403), + ) + + # A cluster that has not applied the events RBAC rule must keep working. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + assert _EVENT_MESSAGE not in status.state_message + assert status.exits[0].reason == "ImagePullBackOff" + + +def test_running_pod_makes_no_events_call(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Running", container_statuses=[_ready_status()]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.RUNNING + assert status.exits == [] + assert api.calls == [] + + +def test_running_pod_with_blocked_sidecar_reports_exit_without_events_call(): + api = _FakeCoreV1Api(events=[_event()]) + + # The verdict stays readiness-based - a serving Pod does not fail because a + # sidecar cannot pull - but the blocked container is still reported. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Running", + container_statuses=[_ready_status(), _waiting_status(name="run-1")], + container_names=["run-0", "run-1"], + ), + ) + status_with_api = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Running", + container_statuses=[_ready_status(), _waiting_status(name="run-1")], + container_names=["run-0", "run-1"], + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.INITIALIZING + assert [e.reason for e in status.exits] == ["ImagePullBackOff"] + assert status_with_api.state == status.state + assert api.calls == [] + + +def test_blocked_init_container_reports_failed_with_component_name(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + init_containers=[kubernetes.client.V1Container(name="init-0")], + init_container_statuses=[_waiting_status(name="init-0")], + annotations={"runtime.gpustack.ai/component-init-0-name": "downloader"}, + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + assert status.exits[0].name == "downloader" + assert status.exits[0].token == "init-0" # noqa: S105 + assert status.exits[0].reason == "ImagePullBackOff" + assert [c[0] for c in api.calls] == ["list_namespaced_event"] + + +def test_blocked_pod_without_api_client_still_reports_failed(): + # The pre-manifest path, and any call site that has no client at hand. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + + +def test_pull_diagnosis_overrides_pod_status_message(): + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + container_statuses=[_waiting_status()], + message="some generic pod message", + ), + ) + + assert "some generic pod message" not in status.state_message + assert "ImagePullBackOff" in status.state_message diff --git a/tests/gpustack_runtime/deployer/test_podman_status.py b/tests/gpustack_runtime/deployer/test_podman_status.py new file mode 100644 index 0000000..f3cbcdd --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_podman_status.py @@ -0,0 +1,144 @@ +from types import SimpleNamespace + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.podman import PodmanWorkloadStatus + + +def _container( + status: str, + component: str = "run", + name: str = "gpustack-test-run-0", + component_name: str = "run-0", + container_id: str = "abc123", + state: dict | None = None, + restart_policy: str = "no", + restart_count: int | None = None, + created: str = "2026-07-29T08:00:00.000000Z", +) -> SimpleNamespace: + attrs = { + "Id": container_id, + "Created": created, + "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, + "State": state if state is not None else {}, + } + if restart_count is not None: + attrs["RestartCount"] = restart_count + + return SimpleNamespace( + name=name, + status=status, + attrs=attrs, + labels={ + "runtime.gpustack.ai/component": component, + "runtime.gpustack.ai/component-name": component_name, + }, + ) + + +def test_podman_workload_status_exit_oomkilled_preserves_state(): + # A container exited 137 with OOMKilled true: the existing state verdict + # (FAILED, no restart policy) is preserved, and an exit entry carries the + # exit code plus the OOMKilled reason. + c = _container( + status="exited", + state={ + "ExitCode": 137, + "OOMKilled": True, + "Error": "", + "StartedAt": "2026-07-29T08:00:00.000000Z", + "FinishedAt": "2026-07-29T08:05:00.000000Z", + }, + restart_count=2, + ) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "run-0" + assert exit_.token == "abc123" # noqa: S105 + assert exit_.exit_code == 137 + assert exit_.reason == "OOMKilled" + assert exit_.started_at == "2026-07-29T08:00:00.000000Z" + assert exit_.finished_at == "2026-07-29T08:05:00.000000Z" + assert exit_.restart_count == 2 + assert status.state_message == "OOMKilled" + + +def test_podman_workload_status_running_container_no_exit_entry(): + # A running container contributes no exit entry. + c = _container(status="running", state={"ExitCode": 0}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits == [] + + +def test_podman_workload_status_exit_missing_state_keys_degrades(): + # A container whose State lacks the keys degrades to an entry with the + # exit code alone rather than raising. + c = _container(status="exited", state={"ExitCode": 1}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + # The State is Docker-compatible, so both backends derive the reason from the + # code and agree with what Kubernetes reports for the same event. + assert exit_.reason == "Error" + assert exit_.message == "" + assert exit_.started_at == "" + assert exit_.finished_at == "" + assert exit_.restart_count == 0 + assert status.state_message == "Error" + + +def test_podman_workload_status_exit_ignores_gos_zero_time(): + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + ) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "" + assert status.exits[0].finished_at == "" + + +def test_podman_workload_status_exit_covers_a_restarting_container(): + c = _container(status="restarting", state={"ExitCode": 1}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 1 + + +def test_podman_workload_status_exit_includes_init_containers(): + # Init containers get an exit entry too, not just run containers. + init_c = _container( + status="exited", + component="init", + name="gpustack-test-init-0", + component_name="init-0", + container_id="init-abc", + state={"ExitCode": 1, "Error": "context deadline exceeded"}, + ) + run_c = _container(status="running", state={}) + + status = PodmanWorkloadStatus(name="test", d_containers=[init_c, run_c]) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "init-0" + assert exit_.token == "init-abc" # noqa: S105 + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "context deadline exceeded" diff --git a/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py b/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py new file mode 100644 index 0000000..4001a2c --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py @@ -0,0 +1,349 @@ +# The cases below drive the deployers' own plan-to-container conversion, +# which is where the device ordering pin becomes observable without a live +# Docker daemon, Podman socket or Kubernetes cluster. +# ruff: noqa: SLF001 + +from types import SimpleNamespace + +import docker.errors +import kubernetes.client +import podman.errors +import pytest + +from gpustack_runtime import envs +from gpustack_runtime.deployer.__types__ import ( + Container, + ContainerEnv, + ContainerExecution, + ContainerResources, + Deployer, + DevicesMaterial, +) +from gpustack_runtime.deployer.docker import DockerDeployer, DockerWorkloadPlan +from gpustack_runtime.deployer.kuberentes import ( + KubernetesDeployer, + KubernetesWorkloadPlan, +) +from gpustack_runtime.deployer.podman import PodmanDeployer, PodmanWorkloadPlan +from gpustack_runtime.detector import ManufacturerEnum + +_ORDERING_ENV = "CUDA_DEVICE_ORDER" + +_NVIDIA_MATERIALS = { + "NVIDIA_VISIBLE_DEVICES": DevicesMaterial( + manufacturer=ManufacturerEnum.NVIDIA, + runtime_env="NVIDIA_VISIBLE_DEVICES", + backend_env=["CUDA_VISIBLE_DEVICES"], + cdi="nvidia.com/gpu", + runtime_values={"0": "0", "1": "1"}, + backend_values={"CUDA_VISIBLE_DEVICES": {"0": "0", "1": "1"}}, + ), +} +_AMD_MATERIALS = { + "AMD_VISIBLE_DEVICES": DevicesMaterial( + manufacturer=ManufacturerEnum.AMD, + runtime_env="AMD_VISIBLE_DEVICES", + backend_env=["HIP_VISIBLE_DEVICES"], + cdi="amd.com/gpu", + runtime_values={"0": "0"}, + backend_values={"HIP_VISIBLE_DEVICES": {"0": "0"}}, + ), +} +_MIXED_MATERIALS = {**_NVIDIA_MATERIALS, **_AMD_MATERIALS} + + +def _deployer(cls, materials: dict[str, DevicesMaterial]): + # Bypass the deployer's __init__, which reaches out to a live daemon or + # API server, and pin the materials so _prepare() short-circuits instead + # of detecting devices. + deployer = object.__new__(cls) + Deployer.__init__(deployer, "test") + deployer._materials = materials + return deployer + + +def _container( + resources: dict, + privileged: bool = False, + declared_envs: dict[str, str] | None = None, +) -> Container: + container_resources = ContainerResources() + container_resources.update(resources) + return Container( + name="default", + image="gpustack/runner:latest", + execution=ContainerExecution(privileged=privileged), + envs=[ContainerEnv(name=n, value=v) for n, v in (declared_envs or {}).items()], + resources=container_resources, + ) + + +class _FakeContainers: + """ + Stand-in for the Docker/Podman clients' container collection. + """ + + def __init__(self, not_found, created: list[dict]): + self._not_found = not_found + self._created = created + + def get(self, name): + raise self._not_found(name) + + def create(self, **kwargs): + self._created.append(kwargs) + return SimpleNamespace(name=kwargs.get("name")) + + +def _docker_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, +) -> list[tuple[str, str]]: + # Keep the requested devices on the env injection path, + # so no CDI specification is generated. + monkeypatch.setattr( + envs, + "GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY", + "Env", + ) + + created: list[dict] = [] + deployer = _deployer(DockerDeployer, materials) + deployer._client = SimpleNamespace( + containers=_FakeContainers(docker.errors.NotFound, created), + ) + deployer._get_image = lambda *_args, **_kwargs: container.image + deployer._mutate_create_options = lambda create_options: create_options + + workload = DockerWorkloadPlan(name="test", containers=[container]) + workload.validate_and_default() + deployer._create_containers(workload, {}, SimpleNamespace(id="pause")) + + return list(created[0]["environment"].items()) + + +def _podman_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, +) -> list[tuple[str, str]]: + # Podman always requests devices via CDI, + # skip generating the CDI specification. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_PODMAN_CDI_SPECS_GENERATE", False) + + created: list[dict] = [] + deployer = _deployer(PodmanDeployer, materials) + deployer._client = SimpleNamespace( + containers=_FakeContainers(podman.errors.NotFound, created), + ) + deployer._get_image = lambda *_args, **_kwargs: container.image + deployer._mutate_create_options = lambda create_options: create_options + + workload = PodmanWorkloadPlan(name="test", containers=[container]) + workload.validate_and_default() + deployer._create_containers(workload, {}, SimpleNamespace(id="pause")) + + return list(created[0]["environment"].items()) + + +def _kubernetes_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, + policy: str = "env", +) -> list[tuple[str, str]]: + monkeypatch.setattr( + "gpustack_runtime.deployer.kuberentes.get_resource_injection_policy", + lambda: policy, + ) + # Resolving the RuntimeClass reads the cluster. + monkeypatch.setattr( + "gpustack_runtime.deployer.kuberentes._resolve_runtime_class_name", + lambda *_args: None, + ) + + class _FakeCoreV1Api: + def __init__(self, client=None): + pass + + def read_namespaced_pod(self, name, namespace): + raise kubernetes.client.exceptions.ApiException(status=404) + + def create_namespaced_pod(self, namespace, body): + return body + + monkeypatch.setattr(kubernetes.client, "CoreV1Api", _FakeCoreV1Api) + + deployer = _deployer(KubernetesDeployer, materials) + deployer._client = None + deployer._node_name = None + deployer._image_pull_secret = None + deployer._mutate_create_pod = lambda pod: pod + + workload = KubernetesWorkloadPlan( + name="test", + namespace="default", + containers=[container], + ) + workload.validate_and_default() + pod = deployer._create_pod(workload, {}) + + return [(e.name, e.value) for e in pod.spec.containers[0].env] + + +@pytest.mark.parametrize( + "name, materials, runtime_envs, expected", + [ + ( + "NVIDIA", + _NVIDIA_MATERIALS, + ["NVIDIA_VISIBLE_DEVICES"], + {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}, + ), + ( + "non-NVIDIA", + _AMD_MATERIALS, + ["AMD_VISIBLE_DEVICES"], + {}, + ), + ( + "mixed manufacturers", + _MIXED_MATERIALS, + ["NVIDIA_VISIBLE_DEVICES", "AMD_VISIBLE_DEVICES"], + {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}, + ), + ( + "unknown runtime visible devices env", + _NVIDIA_MATERIALS, + ["UNKNOWN_RUNTIME_VISIBLE_DEVICES"], + {}, + ), + ( + "no runtime visible devices env", + _NVIDIA_MATERIALS, + [], + {}, + ), + ], +) +def test_map_visible_devices_ordering(name, materials, runtime_envs, expected): + deployer = _deployer(DockerDeployer, materials) + actual = deployer.map_visible_devices_ordering(runtime_envs) + assert actual == expected, f"case {name} expected {expected}, but got {actual}" + + +@pytest.mark.parametrize( + "runner", + [ + _docker_container_envs, + _podman_container_envs, + _kubernetes_container_envs, + ], + ids=["docker", "podman", "kubernetes"], +) +@pytest.mark.parametrize( + "name, materials, resources, privileged, declared_envs, expected", + [ + ( + "all devices", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "all"}, + False, + None, + ["PCI_BUS_ID"], + ), + ( + "specific devices, privileged", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "0"}, + True, + None, + ["PCI_BUS_ID"], + ), + ( + "specific devices, unprivileged", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "0"}, + False, + None, + [], + ), + ( + "all devices, non-NVIDIA", + _AMD_MATERIALS, + {"amd.com/devices": "all"}, + False, + None, + [], + ), + ( + "all devices, ordering declared by the container", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "all"}, + False, + {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"}, + ["FASTEST_FIRST"], + ), + ( + "all devices, auto-mapped on a mixed-manufacturer node", + _MIXED_MATERIALS, + {"gpustack.ai/devices": "all"}, + False, + None, + ["PCI_BUS_ID"], + ), + ], +) +def test_visible_devices_ordering_injection( + name, + materials, + resources, + privileged, + declared_envs, + expected, + runner, + monkeypatch, +): + container_envs = runner( + materials, + _container(resources, privileged, declared_envs), + monkeypatch, + ) + actual = [v for n, v in container_envs if n == _ORDERING_ENV] + assert actual == expected, f"case {name} expected {expected}, but got {actual}" + + +@pytest.mark.parametrize( + "name, resources, expected", + [ + ( + "all devices", + {"nvidia.com/devices": "all"}, + ["PCI_BUS_ID"], + ), + ( + "specific devices", + {"nvidia.com/devices": "0"}, + [], + ), + ], +) +def test_visible_devices_ordering_injection_with_device_plugin( + name, + resources, + expected, + monkeypatch, +): + # Under the KDP injection policy the devices are allocated by a device + # plugin, so a privileged request loses its privilege: an "all" request + # still receives every card and gets the ordering pinned, a specific + # request does not. + container_envs = _kubernetes_container_envs( + _NVIDIA_MATERIALS, + _container(resources, privileged=True), + monkeypatch, + policy="kdp", + ) + actual = [v for n, v in container_envs if n == _ORDERING_ENV] + assert actual == expected, f"case {name} expected {expected}, but got {actual}" diff --git a/tests/gpustack_runtime/deployer/test_workload_status.py b/tests/gpustack_runtime/deployer/test_workload_status.py index e2eb906..3e92068 100644 --- a/tests/gpustack_runtime/deployer/test_workload_status.py +++ b/tests/gpustack_runtime/deployer/test_workload_status.py @@ -1,9 +1,14 @@ +import json from datetime import datetime, timezone import kubernetes.client from gpustack_runtime import envs -from gpustack_runtime.deployer.__types__ import WorkloadStatus +from gpustack_runtime.deployer.__types__ import ( + WorkloadStatus, + WorkloadStatusExit, + WorkloadStatusStateEnum, +) from gpustack_runtime.deployer.kuberentes import ( KubernetesWorkloadStatus, _pin_pod_for_kueue, @@ -63,6 +68,76 @@ def test_workload_status_annotations_json_roundtrip(): assert restored.annotations == status.annotations +def test_workload_status_exits_default_empty(): + # Deployers that report no terminated container leave the list empty, + # so consumers can iterate it without a None check. + status = WorkloadStatus(name="test", created_at="2026-07-29T08:00:00.000000Z") + assert status.exits == [] + + +def test_workload_status_no_exits_json_roundtrip(): + status = WorkloadStatus(name="test", created_at="2026-07-29T08:00:00.000000Z") + restored = WorkloadStatus.from_json(status.to_json()) + assert restored.exits == [] + + +def test_workload_status_exits_json_roundtrip(): + exits = [ + # A terminated container, as Docker/Podman report it. + WorkloadStatusExit( + name="run-0", + token="a1b2c3", # noqa: S106 + exit_code=137, + reason="OOMKilled", + message="", + started_at="2026-07-29T08:00:00.000000Z", + finished_at="2026-07-29T08:05:00.000000Z", + restart_count=2, + ), + # A container blocked from starting, as Kubernetes reports it: + # a reason, and no exit code at all. + WorkloadStatusExit( + name="run-1", + token="run-1", # noqa: S106 + reason="ImagePullBackOff", + message='Back-off pulling image "does-not-exist.invalid/x:y"', + ), + ] + status = WorkloadStatus( + name="test", + created_at="2026-07-29T08:00:00.000000Z", + exits=exits, + ) + + restored = WorkloadStatus.from_json(status.to_json()) + + assert restored.exits == exits + assert all(isinstance(e, WorkloadStatusExit) for e in restored.exits) + assert restored.exits[1].exit_code is None + + +def test_workload_status_exits_absent_in_legacy_payload(): + # A payload serialized before the exit list existed must keep deserializing. + legacy = json.dumps( + { + "name": "test", + "created_at": "2026-07-29T08:00:00.000000Z", + "namespace": None, + "labels": {}, + "annotations": {}, + "state_message": "", + "executable": [], + "loggable": [], + "state": "Unknown", + }, + ) + + restored = WorkloadStatus.from_json(legacy) + + assert restored.exits == [] + assert restored.state == WorkloadStatusStateEnum.UNKNOWN + + def _pinning_pod(labels=None) -> kubernetes.client.V1Pod: return kubernetes.client.V1Pod( metadata=kubernetes.client.V1ObjectMeta(name="p", labels=labels or {}), From fd866ac8f619ed0fd8f2d5b50752e6480701c984 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sat, 15 Aug 2026 11:54:18 +0800 Subject: [PATCH 12/12] test(detector): guard the shared contract, refresh the samples, record the design - add `test_detector_types.py` and `test_detector_cli.py`, covering the ABC composition, the usage merge and `detect --no-usage`: a `Detector` subclass missing either half fails to instantiate, no detector in the shipped package can emit a `vgpu` key, and the omitted columns render as N/A - give the merge tests distinct per-card values and switch health checks on: with identical payloads a positional join, a reversed mapping and a broadcast-to-all all satisfied them, and with health checks off both queries answer HEALTHY without a driver call, so neither test could fail - align every captured sample with the Device contract -- the retired `vgpu` key gone, the NVIDIA and THead samples carrying the `mig` flag, the MIG sample rebuilt around its cards with the instances inside `appendix.mig_devices` -- and add the guard that deserializes each one and compares its keys against `Device`'s fields, since `from_dict` alone swallows an unknown key silently - refresh the samples from the five hardware environments: 2x AMD RX 7800 XT, 1x RTX 5090 D, 2x RTX 4090 48G, 8x Ascend 910B2 and 16x T-Head PPU-ZW810E, replacing the hand-alignments where a real reading differs, and add the 48G-modded 4090 as its own sample since the existing rtx4090d one is a different 24G card - say in the README which samples are measurements and which are not, and why the H100 MIG and Hygon K100 AI ones could not be refreshed - record the design: the goals, the per-vendor parity audit, the query split, the retired physical-index switch, the exit-status surface, the hardware validation and the items deliberately deferred to their own tasks Signed-off-by: thxCode --- ...ctor-alignment-and-workload-exit-status.md | 1088 +++++++++++++++++ .../detector/samples/README.md | 43 + .../samples/detect_output_amd_mi300x.json | 2 - .../samples/detect_output_amd_mi308x.json | 8 - .../samples/detect_output_amd_rx7800xt.json | 47 +- .../samples/detect_output_ascend_310p3.json | 2 - .../samples/detect_output_ascend_910b2.json | 192 +-- .../samples/detect_output_hygon_k100ai.json | 8 - .../samples/detect_output_metax_c500.json | 8 - .../samples/detect_output_nvidia_gb10.json | 2 +- .../samples/detect_output_nvidia_h100.json | 16 +- .../detect_output_nvidia_h100_mig.json | 279 +++-- .../samples/detect_output_nvidia_h200.json | 16 +- .../detect_output_nvidia_rtx4080super.json | 4 +- .../detect_output_nvidia_rtx4090_48g.json | 54 + .../detect_output_nvidia_rtx4090d.json | 2 +- .../detect_output_nvidia_rtx5090d.json | 17 +- .../samples/detect_output_thead_ppu.json | 398 +++++- .../samples/topology_output_amd_rx7800xt.json | 10 +- .../topology_output_nvidia_rtx4090_48g.json | 27 + .../samples/topology_output_thead_ppu.json | 390 +++++- .../detector/test_detector_cli.py | 162 +++ .../detector/test_detector_types.py | 422 +++++++ .../gpustack_runtime/detector/test_samples.py | 97 ++ 24 files changed, 2973 insertions(+), 321 deletions(-) create mode 100644 specs/2026-08-14-detector-alignment-and-workload-exit-status.md create mode 100644 tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json create mode 100644 tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json create mode 100644 tests/gpustack_runtime/detector/test_detector_cli.py create mode 100644 tests/gpustack_runtime/detector/test_detector_types.py create mode 100644 tests/gpustack_runtime/detector/test_samples.py diff --git a/specs/2026-08-14-detector-alignment-and-workload-exit-status.md b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md new file mode 100644 index 0000000..027bec4 --- /dev/null +++ b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md @@ -0,0 +1,1088 @@ +# Spec: Align runtime detector with the operator's device manager, and surface workload exit status + +Status: Built +Type: Feature + +## Summary + +`gpustack_runtime.detector` and `gpustack-operator/pkg/devicemanager/detector` describe the same +hardware through two independently-grown code paths, and the runtime side has drifted: it misses +capability queries the operator performs, it classifies cards as virtual/vGPU, it fetches +utilization metrics it cannot avoid paying for, and it indexes devices by driver minor number — +a scheme that conflicts with `CUDA_DEVICE_ORDER=PCI_BUS_ID` +([gpustack/gpustack#6041](https://github.com/gpustack/gpustack/issues/6041)). Separately, a +`Workload` never reports why a container died, so a Kubernetes deployment whose image cannot be +pulled stays silent in the UI ([gpustack/gpustack#5869](https://github.com/gpustack/gpustack/issues/5869)). + +This change realigns the detect path with the operator vendor by vendor, drops the vGPU notion in +favour of whole-card reporting (MIG instances stay in the card's `appendix`), splits detection into +an information query and a usage query so callers can skip metrics, retires the physical-index +environment switch and pins `CUDA_DEVICE_ORDER=PCI_BUS_ID` on the NVIDIA workloads that can see +every card of the host, and exposes structured per-container exit status — including Kubernetes +image-pull failures backed by Pod Events. + +## Motivation + +### Goals + +- **Audited detect-path parity.** For every one of the nine supported manufacturers, the runtime's + detect path performs the same driver queries, version fallbacks and eligibility filters as the + operator's, or records a deliberate, documented divergence. Target users: GPUStack worker + operators who expect `gpustack-runtime detect` and the operator's `Devices` object to describe the + same card identically. +- **Whole-card inventory.** Detection reports the card the driver enumerates, with no + virtual/vGPU/PF/VF classification. MIG instances of a MIG-enabled card keep their current + representation inside `Device.appendix["mig_devices"]`. +- **Metrics are opt-out.** A caller that only needs inventory (name, uuid, total memory, cores, + versions, topology keys) can obtain it without paying for utilization, temperature or power + queries — which on NVIDIA alone cost a 100 ms GPM sampling window per card. +- **Two composable queries.** Detection is split into an information query and a usage query; + the default keeps today's behaviour by running both and merging the result. +- **Predictable device ordering.** `Device.index` is the detector's enumeration index. The + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` switch is removed; driver-physical numbering + that device-node paths need moves into the appendix. A container that can see every card of the + host — because it is privileged, or because it requested all devices — gets its manufacturer's + ordering pinned by default (`CUDA_DEVICE_ORDER=PCI_BUS_ID` on NVIDIA), so the runtime, the vendor + tooling and the workload all number the cards the same way. +- **Diagnosable workloads.** `WorkloadStatus` carries per-container exit information on Docker, + Podman and Kubernetes; a Kubernetes workload whose image cannot be pulled reports `Failed` with + the ErrImagePull/ImagePullBackOff reason and the corresponding Pod Event message. + +Success is testable: `make test` green, with new tests covering the split API, the removal of the +vGPU classification, the appendix physical indexes, the env propagation, and the exit-status / +image-pull parsing (fixture-driven, no hardware required). + +### Non-Goals + +- Aligning `get_topology` / topology data between the two code bases. Explicitly out of scope. +- Changing how the operator partitions MIG or how it reports devices; the operator is the reference, + not the subject. +- Introducing a metrics/monitor loop in the runtime equivalent to the operator's + `Detector.Start`. The runtime exposes the queries; scheduling stays with the caller. +- Supporting Ascend vNPU (`/dev/vdavinci`) detection. It is removed with the vGPU logic. +- Changing `Device`'s existing field names or the JSON shape of already-reported fields, beyond the + additions and removals listed below. + +## Proposal + +The runtime's detector becomes a faithful, worker-side twin of the operator's device manager for +everything on the detect path, with a query split that lets callers choose how much they pay for. +The deployer stops relying on driver minor numbers for ordering, and starts reporting why a +container is not running. + +### User Stories + +#### Story 1 +As a GPUStack worker operator, I want `gpustack-runtime detect` to report the same card name, memory +size and health as the operator's device manager on the same node, so that a discrepancy between +the two is a real hardware event and not a code-path difference. + +#### Story 2 +As a GPUStack worker operator running MetaX cards on a virtualization-enabled host, I want the +detector to report my physical cards, so that the worker does not come up with zero devices. + +#### Story 3 +As a caller of the runtime library that only needs an inventory (for example to build CDI specs), I +want to ask for device information without utilization metrics, so that a detect pass does not spend +a GPM sampling window per card. + +#### Story 4 +As a caller that renders a live device dashboard, I want to refresh only the usage fields of devices +I already know, so that I can poll cheaply. + +#### Story 5 +As a GPUStack user deploying a model onto all of a node's NVIDIA cards, I want CUDA inside the +inference container to number the GPUs the way `nvidia-smi` and the runtime's detection do, so that +"GPU 2" means the same card everywhere and I do not have to configure anything to get it. + +#### Story 6 +As a GPUStack user deploying a model on a Kubernetes cluster with an unpullable backend image, I +want the deployment to report the pull failure and its reason, so that I understand why the model +never becomes ready instead of watching it hang. + +#### Story 7 +As a GPUStack user whose inference container crashed, I want the workload status to tell me the exit +code and the termination reason (`OOMKilled`, `Error`, signal), so that I can act without shelling +into the node. + +### Core Features & Acceptance Criteria + +#### F1 — Detect-path parity audit and fixes + +Compared method-by-method against the operator (`DetectAccelerator` only; `MonitorAccelerator` is +covered by F3). Known gaps, each to be closed or documented as a deliberate divergence: + +| Vendor | Gap in the runtime today | Expected | +| --- | --- | --- | +| NVIDIA | No GDDR ECC capacity restore. Operator reads `GetMemoryBusWidth` + `GetEccMode` and restores `memory * 16 / 15` when the bus is `< 1024`-bit and ECC is on. | **Withdrawn during PR review, and the divergence kept deliberately.** The restore was implemented in `T5`, then removed: the operator's corrected figure is a *display* value that takes no part in allocation, while `memory` here does, and `memory - memory_used` has to mean free space. Adding back capacity that is not reachable would over-commit every GDDR card with ECC on. `memory` is therefore what the driver reports, i.e. what the card can allocate. | +| NVIDIA | `nvmlDeviceGetMemoryInfo` called without a version preference; operator prefers V2 with a V1 fallback. | Prefer the v2 memory structure, fall back to v1, matching the operator's `GetMemoryInfoV`. The argument is the **packed struct-version constant** `pynvml.nvmlMemory_v2`, not the literal `2` — `pynvml` assigns it straight into the struct's `version` field, which the driver validates, so a literal is rejected. Probe for the constant rather than raising the dependency floor. | +| NVIDIA | `nvmlDeviceGetPciInfo` called without a version preference. | **No change, recorded as a documented divergence.** `pynvml` exposes no v2 accessor at all: `nvmlDeviceGetPciInfo` is an alias of `nvmlDeviceGetPciInfo_v3`, and `nvmlPciInfo_v2_t` is declared but unused. v3 is a superset of the operator's preferred v2, and the operator does not read the struct's string anyway — its `GetBusId()` formats the BDF from domain/bus/device, so the struct version is invisible to what either side consumes. Reaching v2 would mean hand-rolling the struct inside the detector, against this repo's "`pynvml` is a thin wrapper over the upstream package" constraint. | +| NVIDIA | Falls back to *host* memory (`get_memory()`) when the device total reads 0; the operator skips such a device. | Keep, and record it as a deliberate divergence with the reason in a code comment (WSL/iGPU tolerance). | +| Ascend | No device-type filter; the operator skips a device whose `GetType() != NPU_TYPE`. | Skip non-NPU devices, matching the operator's exact semantics: skip **only** when the type call succeeds and the type differs; a device whose type is unreadable is kept. | +| Ascend | Two divergences found while building `T6`, both deliberately left unfixed and carried to the **Deferred** table below. **(a)** The operator's `MonitorAccelerator` reads utilization through a v2→v1 fallback — `dcmi_get_device_utilization_rate_v2` fills a `MultiUtilizationInfo` in one call, else four separate `dcmi_get_device_utilization_rate` calls. `pydcmi` binds no `_v2` utilization at all and the runtime reads only the AICORE rate, so it is permanently on the operator's V1 path. **(b)** `_get_device_memory_status` tries HBM then DDR ECC unconditionally, where the operator picks the ECC device type from whichever memory query succeeded. | Both recorded, neither implemented: **(a)** is outside the four fallbacks `F1` names for Ascend and needs a `pydcmi` addition of its own; **(b)** is pre-existing behaviour and out of `T6`'s scope. | +| Ascend | `*_v2`/`*_v3` DCMI calls with no V1 fallback (vdie, PCIe, chip info, memory); the operator falls back. | Add the V1 fallbacks so older drivers still detect. **Three of the four need entry points `pydcmi` does not bind yet** — `dcmi_get_device_die`, `dcmi_get_device_pcie_info` and `dcmi_get_device_memory_info_v2`, plus the `c_dcmi_pcie_info` / `c_dcmi_memory_info` structs — so `T6` owns `pydcmi/__init__.py` to add them raw, keeping the fallback *policy* in `ascend.py` where the fake can prove it. On the memory fallback, neither struct carries `memory_available`: report `used = memory_size * utiliza / 100` from the utilization percent rather than mirroring the operator, whose `Memory_available == 0` makes `memory_size - available` report **every card as 100 % used** on that path. Verified against `npu-smi` at `C2`. | +| MetaX | **`continue`s on `MXSML_VIRTUALIZATION_MODE_PF`** — i.e. it drops the physical function, the whole card — where the operator drops `VF`. | No virtualization-mode filter at all (see F2); the physical cards are reported. | +| MThreads | Skips `MTML_VIRT_ROLE_HOST_VIRTDEVICE` when `mpcCap != MPC_TYPE_INSTANCE`; the operator skips `GUEST_VIRTDEVICE`. **Worse than a wrong filter: `mpcCap` is not a field of `c_mtmlDeviceProperty_t`** — the real names are `virtCapability, virtRole, mpcCapability, mpcType, rsvd`, and `_PrintableStructure` defines no `__getattr__`, so the read raises `AttributeError`, which `except Exception: raise` propagates and **the whole MThreads detect pass fails**. A non-virtualized card short-circuits on the `and`, which is why it has never been hit. Also note the constant compared against, `MTML_MPC_TYPE_INSTANCE`, belongs to `mpcType`, not to a capability field. | No virtRole filter at all (see F2), which removes the crash and the drop together. **Separately, and deliberately not fixed by `T9`:** `detect_info` frees the device handle in its `finally` (`mthreads.py:135`) and then uses it to open the memory context (`:142`) — a use-after-free on the MTML handle that predates this work. `T9` leaves it untouched because fixing it is a behaviour change outside its ask, and its new `detect_usage` does not reproduce it (it holds the handle open across both contexts and frees it once). **Needs its own task**, which can also carry `mtmlDeviceGetPowerUsage`'s unconverted raw value. | +| Iluvatar | Memory read without the V2→V1 fallback the operator performs. | Add the fallback. | +| Hygon / AMD | Device name resolved from HSA/ASIC only; the operator prefers a `pci.ids` lookup (`GetPCIDeviceNames`) first, then HSA `ProductName`, then the amdgpu marketing name. | The operator's full precedence, **including the libdrm step**: `pci.ids` → HSA `ProductName` → `amdgpu_get_marketing_name` → ASIC market name (AMD) / `rsmi_dev_name_get` (Hygon). The libdrm call was originally left unbound and recorded as a divergence; PR review asked for it, so `pyamdgpu` now binds it and both detectors reach for it. Verified against libdrm on the AMD host, which answers `AMD Radeon RX 7800 XT` for both cards. | +| Cambricon | Not a binding at all: a `cnmon info -e -m -u -j` shell-out parsing `cnmon_info.json`, with a `TODO(thxCode)` where the sample output should be. No driver version, no Neuware version, no cores, no power, no BDF, no NUMA, no health/ECC, no PCIe bus id. | New hand-written `pycndev` ctypes binding (following `pydcmi`/`pymxsml`) and a rewritten `cambricon.py` calling `GetDeviceCount`, `GetDeviceHandleByIndex`, `GetUUID`, `GetPCIeInfoV`, `GetCardName`, `GetMemoryInfoV`, `GetCardHealthStateV`, `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, `GetPowerInfo`, plus the Neuware version from `/usr/local/neuware/version.txt`. `cndev.h` in the operator's `binding/cndev` is the header of record. Four decisions taken while building `T3`, all recorded rather than assumed: **cores and the power limit stay unreported** — `pycndev` binds no core-count entry point and the operator reads neither for this vendor, so closing those two needs a binding addition of its own; **`driver_version` carries `major.minor.build`** where the operator formats only `major.minor`, since it is free precision from the same query and every other vendor here reports the driver's full string; **health comes from `cndevGetCardHealthStateV2().health` alone, with no ECC read**, because that is literally what the operator does (`memoryUnhealthy = healthInfo.Health == 0`); and **memory needs no unit conversion** — the header says MB but the operator, `cnmon` and this repo's Ascend detector all treat the value as MiB, so converting would under-report ~4.8 % against the operator on the same host, which is the very discrepancy Story 1 exists to remove. `C2` confirms the memory figure against `cnmon`. | +| Cambricon | Per-device failure policy diverges from every sibling **on purpose**: `T3` follows the operator and `continue`s past a card whose required reads fail, where the other eight detectors let the error propagate. | A single faulty card therefore costs one device on Cambricon and **all** devices on the other eight. Graceful degradation is very likely the right behaviour everywhere and the other eight should move toward it, but that is a repo-wide behaviour change deserving its own task — not something to slip in per vendor. | +| THead | Enumerates MIG-style GPU/compute instances, which the operator does not. | Keep — this is the appendix mechanism F2 preserves. | +| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. | + +Acceptance criteria: +- ~~A written parity table lives in the repo (module docstring or `docs/`) listing, per vendor, the + driver calls the detect path makes and any deliberate divergence from the operator, with its + reason.~~ **Withdrawn after the build, by request:** `T11` wrote `docs/detector-parity.md` and it + was published, then dropped as not belonging in the shipped documentation. Nothing it recorded is + lost — the deferred items it carried are listed under **Deferred** below, and every divergence it + explained is stated in a comment at the code that makes it. +- Every gap above is either fixed or annotated as deliberate; no gap is left silently open. +- Sample/fixture-driven tests assert that NVIDIA memory is reported as the driver reports it, the Ascend non-NPU skip, and + the MetaX card no longer being dropped. + +#### F2 — Whole-card reporting, no vGPU classification + +- `Device.appendix` no longer carries a `vgpu` key, for any vendor. +- All virtual-card detection is removed: NVIDIA's `_is_vgpu` PCI-capability sniff, the Ascend vNPU + (`dcmi` vdev query) branch, the SR-IOV physical-function comparisons in AMD / Hygon / Iluvatar / + THead, MetaX's virtualization-mode check, MThreads' `virtRole` check. +- MIG instances keep the existing mechanism unchanged: the physical card is the reported device, its + instances live in `appendix["mig_devices"]`, numbered by `index_mig_devices`, and + `expand_mig_devices` still substitutes them for callers that address instances. +- `deployer/cdi/ascend.py` always uses `/dev/davinci{index}`; the `/dev/vdavinci` branch is removed. +- No detector filters devices by virtualization mode or role; whatever the driver enumerates is + reported. + +#### F3 — Split information and usage queries + +- `Detector` grows two methods and keeps `detect` as the composing entry point: + - `detect_info() -> Devices | None` — abstract; identity, capability and inventory fields only. + - `detect_usage(devices: Devices | None = None) -> Devices | None` — utilization, used memory, + memory utilization, temperature, used power, plus a refreshed `memory_status`. With `devices` + given it merges into them (matched by `uuid`, MIG entries included); with `None` it runs + `detect_info()` first. + - `detect(usage: bool = True) -> Devices | None` — `detect_info()`, then `detect_usage()` merged in + when `usage` is true. Default behaviour is byte-for-byte what `detect()` returns today. +- Field split: **information** = `manufacturer, index, name, uuid, driver_version, runtime_version, + runtime_version_original, compute_capability, cores, memory, power, appendix`; **usage** = + `cores_utilization, memory_used, memory_utilization, temperature, power_used`, and + `memory_status` is produced by both (mirroring the operator, which reports `Unhealthy` from both + `DetectAccelerator` and `MonitorAccelerator`). +- Module level: `detect_devices(fast=True, manufacturer=None, usage=True)`; the new keyword threads + through to every detector. +- `gpustack-runtime detect` grows `--no-usage`; the table format renders `N/A` for the omitted + columns instead of `0`. +- With `usage=False`, no utilization/temperature/power driver call is made — asserted per vendor by + monkeypatched-binding tests that fail if such a call happens. + +#### F4 — Retire the physical-index switch, propagate `CUDA_DEVICE_ORDER` + +- `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` is removed from `envs.py` and from all three + call sites (NVIDIA, Ascend, Iluvatar). `Device.index` is the detector's enumeration index. +- Driver-physical numbering that a device-node path or a vendor tool needs is exposed in the + appendix instead — mirroring the operator, which keeps a sequential `Index` next to + `PhysicalIndexes`. AMD's `card_id`/`renderd_id` and Ascend's `card_id`/`device_id` already do this; + NVIDIA, Iluvatar and THead gain their minor number the same way — but **only NVIDIA's and + Iluvatar's name a device node.** THead's `/dev/alixpu_ppu{N}` is named after the card ordinal, i.e. + the enumeration index; the operator's own comment at its `GetMinorNumber` call site says so, and it + keeps the number purely to *prove* a node addresses the card it describes, by comparing it against + the node's character-device minor. So THead records `minor_number` in the appendix and its CDI path + keeps reading `Device.index`. +- Every consumer of `dev.index` that means "device node number" (`deployer/cdi/*`) is reviewed + against the appendix values, so no `/dev/...` path changes meaning. Two of them actually read the + retired numbering, because the switch defaults to **on** and therefore describes today's real + behaviour: `cdi/ascend.py`'s `/dev/davinci{...}` (physical id from the DCMI logic id) and + `cdi/iluvatar.py`'s `/dev/iluvatar{...}` (NVML minor number). Both move to the appendix key in the + same change that retires the switch, so no host ever sees a window where a workload is handed + another card's device node. `cdi/amd.py`, `cdi/hygon.py` and `cdi/metax.py` already build their + paths from `appendix["card_id"]` / `["renderd_id"]`, `cdi/thead.py` was never on the switch, and + NVIDIA has no CDI generator in this repository. +- A container that ends up seeing **every** accelerator of the host gets its manufacturer's device + ordering pinned by default, on Docker, Podman and Kubernetes alike. "Every accelerator" is exactly + the condition the deployers already resolve per device request: the container is `privileged`, or + its device request is `all`. On NVIDIA that means `CUDA_DEVICE_ORDER=PCI_BUS_ID`; no other + manufacturer documents an ordering switch, so no other variable is injected. + - It is a **default**, not an override: a container that declares `CUDA_DEVICE_ORDER` itself keeps + its own value. + - A container that was given a specific subset is left alone — it does not see the cards it was + not given, and the deployer already scopes its visible-devices env. The criterion is the subset, + not who allocated it: a device plugin handing out *every* card of the node (an `all` request + under the KDP injection policy) still leaves the container seeing everything, so the ordering is + pinned there too. + - The worker's own environment is irrelevant to this decision. The ordering is a property of what + the deployer hands the container, so the deployer decides it. + - The operator pins the same variable, by the same never-overwrite rule, at a **complementary** + point: its NVIDIA device plugin sets it on *sliced* containers, because HAMi-core fills its + `CUDA_DEVICE_MEMORY_LIMIT_` table in NVML enumeration order but reads a limit back by CUDA + ordinal, and the two coincide only under `PCI_BUS_ID`. So on an operator-managed cluster a sliced + workload gets it from the plugin and a privileged / all-devices workload gets it from here. Same + name, same value, both set-if-absent — the two injections cannot disagree. +- `Device.index`'s docstring loses the physical-index paragraph. +- **A skipped device does not renumber its survivors.** The operator compacts its `Index`, so its + second card becomes `Index 0` when the first is skipped. The runtime keeps the driver's enumeration + index, because compacting would make the index *unstable across passes*: a transiently faulty card + would shift every later card's index down and back again on recovery, and `DevicesMaterial`'s + `runtime_values` / `backend_values` / `numa_affinities` are all keyed by `str(dev.index)`. A + non-contiguous index is a static fact a caller can hold; a shifting one silently repoints a cached + index at a different card. Decided while building `T3`, the only detector that skips a device. + +#### F5 — Workload exit status and Kubernetes image-pull failures + +- `WorkloadStatus` gains a per-container exit list — one entry per container that has terminated or + is blocked from starting — carrying: container name, operation token, exit code, reason, message, + started/finished timestamps, and restart count. Absent containers are simply not listed. +- Docker and Podman populate it from each container's `State` (`ExitCode`, `Error`, `OOMKilled`, + `StartedAt`, `FinishedAt`, `RestartCount`), including init containers. +- Kubernetes populates it from `container_statuses` / `init_container_statuses`, reading + `last_state.terminated` and `state.terminated` for exit codes and `state.waiting` for + reason/message. +- A Kubernetes workload whose image cannot be pulled (`ErrImageNeverPull`, `ErrImagePull`, + `ImagePullBackOff`, `InvalidImageName`, `RegistryUnavailable`) reports `state = Failed` — not + `Pending` — with the reason and message in `state_message`, and the matching Pod `Event` message + appended. `ErrImageNeverPull` was added by the end-of-build review: it is the one genuinely + unrecoverable reason of the five (`imagePullPolicy: Never` with the image absent), and it was + missing while recoverable reasons were present. +- The appended Event is selected by `involvedObject.uid` as well as name, chosen by timestamp rather + than list order — the API guarantees none — and prefers the kubelet's `Failed` Event, so a stale + `FailedScheduling` cannot stand in as the diagnosis. +- Pod Events are read only when the Pod is in such a blocked state, and an Events call denied by + RBAC degrades gracefully (state and reason still reported, Event detail omitted, one debug log). +- `deploy/manifests/kubernetes.yaml` grants `get`/`list` on core `events`. +- Fixture-driven tests cover: Docker exited-nonzero, Docker OOMKilled, K8s terminated exit code, + K8s `ImagePullBackOff` → `Failed` with reason, and an Events call raising `403`. + +### Notes / Constraints / Caveats + +- Python ≥ the project's floor; `uv` for dependency management; `ruff` + `pre-commit` for lint. +- Vendor bindings are hand-written `ctypes` modules under `gpustack_runtime/detector/py*` (see + `pydcmi`, 1.2k lines; `pymxsml`, 1.6k lines) except `pynvml`/`pymtml`, which are thin wrappers over + upstream PyPI packages. `pycndev` follows the hand-written pattern. +- The operator is the reference implementation and is not modified by this work. +- All new fields are `dataclass_json` dataclass fields, defaulted so existing serialized payloads + keep deserializing. +- Hardware-dependent tests stay `pytest.mark.skipif(not .is_supported())`, as today; new + behaviour is covered by fixtures and monkeypatched bindings so CI without GPUs still exercises it. + +### Boundaries + +- **Always:** keep MIG instances in `appendix["mig_devices"]` with the current numbering; keep + `detect()`'s default output identical to today's; keep the topology **data** untouched — a + `get_topology` body may be repointed at `detect_info()` so topology stops paying for metrics it + never reads, but the `Topology` it returns must not change; keep `Device`'s existing field names. +- **Ask first:** before changing what a `/dev/...` device-node path resolves to for any vendor; + before adding an RBAC verb beyond core `events`; before touching the operator repository. +- **Never:** align `get_topology`'s data or semantics with the operator's, or change what it returns; + add a monitor/scheduling loop to the runtime; override a + `CUDA_DEVICE_ORDER` the container already declares; inject an ordering variable into a container + that was given a device subset. + +### Risks and Mitigations + +- Ascend `/dev/davinci{N}` numbering may follow the driver's physical id, not the logic id → keep + the physical id in the appendix and have the CDI generator read it from there; confirm against a + real 910B/910C node before merging. +- Retiring the physical-index switch shifts the *keys* other consumers derive from `Device.index`: + `DevicesMaterial.runtime_values` / `backend_values` / `numa_affinities` are keyed by + `str(dev.index)` (`deployer/__types__.py:1414-1427`), and every CDI generator emits its devices as + `ConfigDevice(name=str(dev.index))`. On NVIDIA and Iluvatar those keys go from the minor number to + the enumeration index, and on Ascend from the physical id to the logic id → harmless within one + process, because a single detect pass feeds both sides, and it is the direction issue #6041 asks + for; but a caller that persisted indexes across the upgrade sees them move, so it belongs in the + release note next to the ordering change. +- Removing the Ascend vNPU branch drops detection on hosts that pre-split NPUs → documented as a + Non-Goal and a release note; partitioning is the operator's device manager's job. +- Removing MetaX's `PF` skip and MThreads' `virtRole` skip changes reported device counts on + virtualization-enabled hosts → sample-driven tests pin the expected inventory for both. +- A new `pycndev` binding cannot be validated without Cambricon hardware → derive it from the + operator's `cndev.h`, unit-test struct layouts and parsing against captured samples, and keep the + hardware path behind `is_supported()`. +- The `usage=False` path could silently keep issuing metric calls → tests assert on the binding + functions actually invoked, not just on the returned values. +- Reading Pod Events adds API traffic per status poll → only for blocked Pods, only field-selected + to the Pod, and skipped entirely once the workload runs. +- Pinning `CUDA_DEVICE_ORDER=PCI_BUS_ID` changes the CUDA ordinals inside a container that previously + ran under the `FASTEST_FIRST` default, so a workload addressing cards by CUDA ordinal sees a + different card for the same number on a heterogeneous host → that is the defect being fixed (the + ordinals now agree with `nvidia-smi` and with the runtime's own detection), but it is a visible + behaviour change on such hosts, so it belongs in the release note; a container that needs the old + behaviour declares `CUDA_DEVICE_ORDER=FASTEST_FIRST` itself and keeps it. +- `T1` mechanically renames `detect` → `detect_info` in all nine vendor files, so it conflicts with any + in-flight branch touching a detector → land it first and alone, before the per-vendor tasks start. +- `samples/` is documentation-only — no test reads it, and it has already drifted from the code + (`detect_output_nvidia_h100_mig.json` still uses the retired standalone-MIG shape with + `gpu_instance_index` / `compute_instance_index`) → `T12` realigns every sample and adds a + schema-guard test, so a future drift fails the suite instead of rotting silently. +- `pytest.ini` lists an `integration_tests` testpath that does not exist → create the directory (with + the integration scenarios below) or drop the entry, before adding integration tests. +- No coverage measurement exists today (`addopts` carries no `--cov`), so the Test Plan's per-package + numbers are targets, not deltas → introduce `pytest-cov` as an optional prerequisite, or accept + hand-review of the new tests' reach. +- Real-hardware validation cannot run during the build (macOS dev machine, no vendor drivers) → every + task verifies through fixtures/monkeypatched bindings, and `C2` gates the merge on the user-provided + environments. + +## Design Details + +### Commands + +**Environment.** Implementation and the whole automated suite run **locally** (macOS dev machine, `uv`). +No vendor driver exists there, so every detector's hardware path is skipped by `is_supported()` and all +new behaviour must be provable through fixtures and monkeypatched bindings — that constraint shapes the +Test Plan. Real-hardware validation is checkpoint `C2`, on the environments below. + +**`C2` environments** (supplied 2026-08-15). The checkout lives at `~/github.com/gpustack/runtime` on +each target and its git tree must be brought in line with the branch under test. + +| Vendor | Access | +| --- | --- | +| THead (ppu) | `root@192.168.100.151`, jumping through `frank@192.168.50.17` | +| AMD | `frank@192.168.50.17` | +| NVIDIA | `frank@192.168.50.13` and `frank@192.168.50.16` | +| Ascend | `root@120.241.57.28` | + +Two ways to get the code running there, per target: + +- **From source**, where the target has Python: `make install`, then run the CLI from the checkout's + `.venv`. +- **From a container image**, built on `frank@192.168.50.17`: + `PACKAGE_NAMESPACE=thxcode PACKAGE_TAG=dev- make package`, pushing `thxcode/runtime:dev-` + to Docker Hub, then running that image on the target. + +```bash +# On each target, per pass +gpustack-runtime detect --format json +gpustack-runtime detect --no-usage --format json +gpustack-runtime topology --format json +# Health only means anything with the ECC read switched on -- see F1's all-vendors row +GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false gpustack-runtime detect --format json +``` + +**One question only the AMD/Hygon host can answer.** Both build their UUID as +`f"GPU-{rsmi_dev_unique_id_get(idx)[2:]}"`, and that wrapper returns `hex(value)`, so a driver +answering `0` yields `"GPU-0"` for *every* card — a truthy but non-unique identity, which is the key +the usage merge joins on. The merge now drops an ambiguous id rather than broadcasting one card's +metrics onto the rest, so the harm is contained either way, but whether the ids are actually distinct +is a measurement, not a guess: compare the `uuid` values `detect --format json` reports on the AMD +host. If they collide, the identity needs a distinct fallback (the BDF is read on the same pass) and +that becomes its own task. + +**These four cover four of the nine vendors**, plus Hygon by proxy (below). Iluvatar, MetaX, MThreads +and Cambricon have no environment, so their detect paths stay proven only by fixtures — and in +particular the MetaX temperature Open Question and Cambricon's "the driver says MB but means MiB" +reading **cannot be settled by `C2` as scoped**. Both must keep their markers until hardware appears. + +**Hygon's logic is exercised on the AMD host, by proxy.** A Hygon DCU *is* a ROCm device, and +`hygon.py` runs on the same binding stack AMD does — `pyrocmsmi` + `pyhsa`, with `pyamdgpu` for the +cores fallback. The only vendor-specific gate is one PCI vendor id: `hygon.py`'s +`get_pci_devices(vendor="0x1d94")` against `amd.py`'s `"0x1002"`. Temporarily pointing Hygon's at +`0x1002` on the AMD host makes `HygonDetector.is_supported()` pass and runs every line of its +`detect_info` / `detect_usage` against a real driver. + +- **What that proves:** the Hygon path executes without raising on real hardware; the `pyrocmsmi` / + `pyhsa` field names it reads exist; its fallback chains resolve; the usage merge and the + `memory_status` recomputation work; and its CDI generator emits paths. +- **What it does not prove:** any Hygon-specific *value*. The names, ECC counters, memory and core + counts will be the AMD card's, so nothing about a K100AI's own reporting is settled by it. +- The edit is a throwaway on the target, recorded in the evidence and reverted; it must never be + committed. **The trick does not generalise** — Iluvatar, MetaX, MThreads and Cambricon each load + their own vendor library, which is absent on these hosts, so there is nothing for a vendor-id swap + to reach. + +```bash +make prepare # REQUIRED FIRST: writes the gitignored gpustack_runtime/_version_appendix.py. + # Without it any `uv run` fails at the editable build with + # "Forced include not found: .../gpustack_runtime/_version_appendix.py". +make deps # uv sync --all-packages && uv lock && uv tree +make lint # uv run pre-commit run --all-files --show-diff-on-failure + # (ruff-check --fix --unsafe-fixes, ruff-format, codespell — it rewrites files) +make test # uv run pytest (pytest.ini: pythonpath=., addopts=--no-header -vvv) +make docs # mkdocs build + +# Task-scoped verification +uv run pytest tests/gpustack_runtime/detector -q +uv run pytest tests/gpustack_runtime/deployer -q +uv run pytest tests/gpustack_runtime/detector/test_samples.py -q + +# Manual smoke on the dev machine (no devices -> empty list, must not traceback) +uv run gpustack-runtime detect --format json +uv run gpustack-runtime detect --no-usage --format json +``` + +### Project Structure + +``` +gpustack_runtime/ +├── detector/ +│ ├── __init__.py # detect_devices/detect_backend/expand_mig_devices facade +│ ├── __types__.py # Device, Devices, Topology, Detector ABC, index_mig_devices +│ ├── __utils__.py # PCI/NUMA/version/unit helpers +│ ├── .py # amd, ascend, cambricon, hygon, iluvatar, metax, mthreads, nvidia, thead +│ └── py/ # bindings. Hand-written ctypes: pyamdgpu, pyamdsmi, pydcmi, pyhgml, +│ # pyhsa, pyixml, pymxsml, pyrocmsmi (+ NEW pycndev). +│ # Thin wrappers over upstream PyPI: pynvml, pymtml. +├── deployer/ +│ ├── __types__.py # Container/Workload{Plan,Status}, Deployer ABC, DevicesMaterial +│ ├── docker.py # DockerWorkloadStatus.parse_state +│ ├── podman.py +│ ├── kuberentes.py # KubernetesWorkloadStatus.parse_state, Pod/Event reads +│ └── cdi/.py # CDI spec generation, consumes Device.index/appendix +├── cmds/detector.py # `detect` / `topology` sub-commands +└── envs.py # GPUSTACK_RUNTIME_* env surface +deploy/manifests/kubernetes.yaml # RBAC for the deployer +tests/gpustack_runtime/ +├── detector/ +│ ├── fixtures/__init__.py # load() helper only — no data files yet +│ ├── samples/ # detect_output_*.json / topology_output_*.json captured from real +│ │ # hardware. Documentation-only today: NO test reads them, so they +│ │ # have drifted (detect_output_nvidia_h100_mig.json still uses the +│ │ # retired standalone-MIG shape). T12 realigns them and adds a guard. +│ └── test_.py # hardware-gated smoke tests (skipif not is_supported()) +└── deployer/ + ├── fixtures/ + └── test_workload_status.py, test_privileged.py, test_runtime_class.py, test_utils.py +``` + +### Code Style + +The query split, as the ABC composes it (`detector/__types__.py`): + +```python +class Detector(ABC): + @abstractmethod + def detect_info(self) -> Devices | None: + """ + Detect devices' inventory, without usage metrics. + """ + raise NotImplementedError + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of the given devices, merged into them in place. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + """ + return devices + + def detect(self, usage: bool = True) -> Devices | None: + devices = self.detect_info() + if usage and devices: + self.detect_usage(devices) + return devices +``` + +A vendor's information query, showing the operator-mirroring comment convention: + +```python +class NVIDIADetector(Detector): + def detect_info(self) -> Devices | None: + """ + Detect NVIDIA GPUs' inventory using pynvml, without usage metrics. + + Returns: + A list of detected NVIDIA GPU devices, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + ret: Devices = [] + try: + pynvml.nvmlInit() + ... + dev_mem = 0 + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo( + dev, + version=pynvml.nvmlMemory_v2, # packed struct version, not a literal 2 + ) + # What the driver reports, i.e. what the card can allocate. The + # operator restores the ~1/16 that ECC parity carves out of a GDDR + # part, but that figure is display-only there, while this one takes + # part in allocation -- see F1's first row. + dev_mem = byte_to_mebibyte(dev_mem_info.total) + except pynvml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices") + raise + return ret +``` + +Conventions: `from __future__ import annotations as __future_annotations__` first; `@dataclass_json +@dataclass` models with per-field docstrings; `contextlib.suppress(Error)` around optional +driver calls; `debug_log_exception` / `debug_log_warning` for diagnostics; Google-style docstrings +with `Args:` / `Returns:` / `Raises:`; comments explain *why*, referencing the operator when the +behaviour is a deliberate mirror. + +### Implementation Plan + +Two independent tracks — the detector track (T*) and the deployer track (D*) share no file and start in +parallel — joined by two checkpoints. The detector track is sequenced **expand → migrate → contract**, +because splitting `detect` is one mechanical change whose blast radius covers all nine vendor files: no +vertical slice can land green while half the vendors still override `detect()` and the other half +implement `detect_info()`. + +#### Detector track + +- [x] **T1 · Prefactor: split the Detector query surface (expand), retire the physical-index switch** + Blocked by: None + Owns: `gpustack_runtime/detector/__types__.py`, `gpustack_runtime/detector/__init__.py`, + `gpustack_runtime/detector/__utils__.py`, `gpustack_runtime/detector/*.py`, + `gpustack_runtime/envs.py`, `gpustack_runtime/deployer/cdi/ascend.py`, + `gpustack_runtime/deployer/cdi/iluvatar.py`, + `tests/gpustack_runtime/detector/test_detector_types.py`, + `tests/gpustack_runtime/detector/test_mig_devices.py` + Gate: review + Scope: add `detect_info()` / `detect_usage(devices=None)` / `detect(usage=True)` to the `Detector` + ABC as shown in Code Style; rename every vendor's `detect()` to `detect_info()` with no body + change, so `detect()` output is unchanged and `usage=False` is an accepted no-op until each + vendor migrates; thread `usage=` through `detect_devices()` and call it with `usage=False` from + `get_devices_topologies()`, and point the nine vendors' own `get_topology()` at `detect_info()` + so topology stops paying for metrics once T3–T10 land (identical body during expand, so no + behaviour change); add a module-level `merge_devices_usage(devices, usages)` to `__types__.py` + joining by `uuid` across cards **and** `appendix["mig_devices"]` entries, mirroring the + operator's `MonitorAccelerator`, which returns a separate UUID-keyed metrics list that consumers + join by identity, never by index — T3–T10 call it from their `detect_usage`; delete + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` from `envs.py` (both the `TYPE_CHECKING` stub + and the `variables` registry entry that actually resolves it) and its three call sites (NVIDIA, + Ascend, Iluvatar), moving the driver-physical number into the appendix (`minor_number` for + NVIDIA/Iluvatar — THead's belongs to T10, whose index the switch never touched; Ascend adds + `physical_id` beside its existing `card_id`/`device_id`); **keep the two `/dev/...` paths that + read the retired numbering byte-for-byte** — `cdi/ascend.py`'s `/dev/davinci{...}` and + `cdi/iluvatar.py`'s `/dev/iluvatar{...}` become one-line reads of the appendix key, since the + switch defaults to **on** and those paths are physical numbering today (T6/T7 then only re-pick + which key); add a `get_pci_device_name()` pci.ids lookup helper to `__utils__.py` for T4; drop + the physical-index paragraph from `Device.index`'s docstring, wording it as the index the + detector enumerates the device at, without claiming contiguity (Ascend reports the DCMI logic + id, and MIG numbering punches holes). + Acceptance: `detect()`'s payload is unchanged **by construction** — no vendor overrides + `detect()`, so it is `detect_info()`'s payload plus a no-op merge; asserted structurally (every + vendor implements `detect_info`, none overrides `detect`, the composition preserves the info + fields) rather than against nine golden fixtures, which would need the `fixtures/bindings.py` + fakes the Test Plan lists as a prerequisite and T3–T10 build. "Unchanged" means every + pre-existing field and appendix key keeps its value, the appendix *gains* the physical-number + key, and an index moves only where F4 intends it to. Plus: `detect(usage=False)` is accepted by + every detector; the env var appears nowhere in the tree; `Device.index` is the enumeration index + and the physical number is reachable from the appendix; `/dev/davinci{N}` and `/dev/iluvatar{N}` + resolve to the same nodes as before the change; `get_pci_device_name()` resolves a known + vendor:device pair, prefers a subsystem name over the device name, and returns `""` on a miss. + Verify: `make prepare && uv run pytest tests/gpustack_runtime/detector -q` + +- [x] **T2 · PoC: `pycndev` ctypes binding** + Blocked by: None + Owns: `gpustack_runtime/detector/pycndev/**`, `tests/gpustack_runtime/detector/test_pycndev.py`, + `ruff.toml` + Gate: review + Scope: hand-written ctypes binding following `pydcmi` / `pymxsml`, derived from the operator's + `binding/cndev/cndev.h` (the header of record): library loader, init/release, `GetDeviceCount`, + handle-by-index, `GetUUID`, `GetPCIeInfoV` (V2), `GetCardName`, `GetMemoryInfoV` (V2), + `GetCardHealthStateV` (V2), `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, + `GetPowerInfo`. + Acceptance: the module imports cleanly with no Cambricon library present and makes no call at + import time; every struct's `ctypes.sizeof` and field offsets match `cndev.h`; a missing library + surfaces as the binding's own error type, not a bare `OSError`. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_pycndev.py -q` + +- [x] **T3 · Cambricon detector on `pycndev`** + Blocked by: T1, T2 + Owns: `gpustack_runtime/detector/cambricon.py`, + `tests/gpustack_runtime/detector/test_cambricon.py` + Gate: review + Scope: replace the `cnmon info -e -m -u -j` shell-out (and its `TODO(thxCode)` placeholder) with + `pycndev`. `detect_info`: uuid, card name, total memory, driver version, Neuware version from + `/usr/local/neuware/version.txt` (`\d+\.\d+\.\d+`, as the operator reads it), PCIe bus id, NUMA + node, health from `GetCardHealthStateV`. `detect_usage`: core utilization, used memory, memory + utilization, temperature, power. Drop `appendix["vgpu"]`. + Acceptance: with a monkeypatched `pycndev`, `detect_info()` returns a Device carrying name, uuid, + memory, driver_version, runtime_version, bdf, numa and a real `memory_status`, and issues no + utilization/temperature/power call; `detect()` adds the usage fields; `cnmon` is invoked nowhere + in the tree. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_cambricon.py -q` + +- [x] **T4 · AMD + Hygon: pci.ids name precedence, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/amd.py`, `gpustack_runtime/detector/hygon.py`, + `gpustack_runtime/deployer/cdi/amd.py`, `gpustack_runtime/deployer/cdi/hygon.py`, + `tests/gpustack_runtime/detector/test_amd.py`, `tests/gpustack_runtime/detector/test_hygon.py` + Scope: adopt the operator's name precedence — pci.ids (`get_pci_device_name` from T1) → HSA + product name → amdgpu marketing name → ASIC market name; delete `appendix["vgpu"]` and the + `get_physical_function_by_bdf` comparison; move `amdsmi_get_gpu_metrics_info`, used power, + `rsmi_dev_busy_percent_get`, `rsmi_dev_temp_metric_get` and used memory into `detect_usage`, + keeping the power *limit* in `detect_info`; confirm the CDI generators read + `appendix["card_id"]` / `["renderd_id"]`, not `Device.index`. + Acceptance: pci.ids hit wins over the HSA name; with pci.ids absent the previous name is + returned; no appendix carries `vgpu`; `detect_info()` makes no utilization/temperature/used-power + call; the CDI spec for both vendors is byte-identical to the pre-change output for the same + fixture. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_amd.py tests/gpustack_runtime/detector/test_hygon.py -q` + +- [x] **T5 · NVIDIA: memory version preference, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/nvidia.py`, `tests/gpustack_runtime/detector/test_nvidia.py`, + `tests/gpustack_runtime/detector/test_mig_devices.py` + Gate: review + Scope: prefer the v2 memory structure with a v1 fallback; delete `_is_vgpu` and + `appendix["vgpu"]`; keep the MIG appendix mechanism, moving the MIG entries' usage fields + (`cores_utilization`, `memory_used`, `memory_utilization`, `temperature`, `power_used`) into + `detect_usage`; keep the host-memory fallback for a zero total and record in a comment that it + is a deliberate divergence from the operator (WSL / iGPU tolerance); record + `appendix["minor_number"]`. + **The GDDR ECC capacity restore this task originally carried was reverted during PR review** + — see F1's first row — so `_restore_ecc_reserved_memory()` and its bus-width/ECC-mode + acceptance cases exist nowhere in the tree; `memory` is what the driver reports. + Acceptance: monkeypatched NVML — the v2 memory structure is preferred and a binding without it + falls back to v1; a GDDR card with ECC on reports the driver's total unrestored; `detect_info()` + issues no GPM/utilization/temperature/power-usage call; `vgpu` absent from every appendix; a + MIG-enabled card still carries `appendix["mig_devices"]` with its instances numbered by + `index_mig_devices`. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_nvidia.py tests/gpustack_runtime/detector/test_mig_devices.py -q` + +- [x] **T6 · Ascend: NPU type filter, V1 fallbacks, drop the vNPU branch, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/ascend.py`, `gpustack_runtime/detector/pydcmi/__init__.py`, + `gpustack_runtime/deployer/cdi/ascend.py`, + `tests/gpustack_runtime/detector/test_ascend.py` + Gate: review + Scope: skip a device whose DCMI type is not NPU, as the operator does; add V1 fallbacks for vdie, + PCIe, chip-info and memory so older drivers still detect; delete the vdev (vNPU) branch and + `appendix["vgpu"]`; make the CDI generator always emit `/dev/davinci{...}` from the appendix + value that matches the driver's device-node numbering — T1 already wired it to `physical_id` to + preserve today's paths, so this is a one-line re-pick if `C2` shows the logic id is the right + key (see Open Questions); move utilization, temperature and power into `detect_usage`. + Acceptance: a non-NPU device in the card is not reported; a driver exposing only the V1 calls + still yields a Device; no `/dev/vdavinci` path is produced anywhere; `detect_info()` makes no + utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_ascend.py -q` + +- [x] **T7 · Iluvatar: memory V2→V1 fallback, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/iluvatar.py`, `gpustack_runtime/deployer/cdi/iluvatar.py`, + `tests/gpustack_runtime/detector/test_iluvatar.py` + Scope: add the memory V2→V1 fallback the operator performs; delete `appendix["vgpu"]` and the + `get_physical_function_by_bdf` comparison; move utilization, temperature and used power into + `detect_usage`; confirm the CDI `/dev/iluvatar{N}` path still reads the appendix minor number + T1 wired it to. + Acceptance: v2 unavailable → v1 result returned; no appendix carries `vgpu`; `detect_info()` + makes no utilization/temperature/used-power call; the CDI spec is unchanged for the same fixture. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_iluvatar.py -q` + +- [x] **T8 · MetaX: remove the inverted PF skip, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/metax.py`, `gpustack_runtime/deployer/cdi/metax.py`, + `tests/gpustack_runtime/detector/test_metax.py` + Gate: review + Scope: delete the `continue` on `MXSML_VIRTUALIZATION_MODE_PF` — it currently drops the physical + function, i.e. the whole card, where the operator drops `VF` — and apply no virtualization-mode + filter at all; delete `appendix["vgpu"]`; move core utilization, used memory, temperature and + board power into `detect_usage`. + Acceptance: a fixture reporting `mode == PF` yields that card as a Device (it is dropped today); + a fixture reporting `mode == VF` also yields a Device; `detect_info()` makes no + utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_metax.py -q` + +- [x] **T9 · MThreads: remove the virtRole skip, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/mthreads.py`, `tests/gpustack_runtime/detector/test_mthreads.py` + Gate: review + Scope: delete the `MTML_VIRT_ROLE_HOST_VIRTDEVICE` / `mpcCap` skip and `appendix["vgpu"]`, and + apply no virtRole filter; move GPU utilization, temperature, used memory and used power into + `detect_usage`. + Acceptance: a fixture reporting a host virt device with `mpcCap != MPC_TYPE_INSTANCE` is now + reported (it is dropped today); `detect_info()` makes no utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_mthreads.py -q` + +- [x] **T10 · THead: de-vGPU, usage split (MIG appendix preserved)** + Blocked by: T1 + Owns: `gpustack_runtime/detector/thead.py`, `gpustack_runtime/deployer/cdi/thead.py`, + `tests/gpustack_runtime/detector/test_thead.py` + Scope: delete `appendix["vgpu"]` (both the card's and the instance entries') and the + `get_physical_function_by_bdf` comparison; keep the GPU/compute-instance enumeration in the + appendix — the operator has no THead equivalent and this is the mechanism F2 preserves; move + utilization, temperature and used power into `detect_usage`, instance entries included; record + `appendix["minor_number"]` where the binding exposes it. Creates `test_thead.py`, which does not + exist today. + Acceptance: no appendix carries `vgpu`; instance entries survive with `sliced` intact (the + topology path keys off it); `detect_info()` makes no GPM/utilization/temperature/used-power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_thead.py -q` + +- [x] **T11 · Contract: make the split mandatory, `detect --no-usage`, ~~parity table~~** + Blocked by: T3, T4, T5, T6, T7, T8, T9, T10 + Owns: `gpustack_runtime/detector/__types__.py`, `gpustack_runtime/detector/__utils__.py`, + `gpustack_runtime/cmds/detector.py`, ~~`docs/**`, `mkdocs.yml`~~, + `tests/gpustack_runtime/detector/test_detector_cli.py`, + `tests/gpustack_runtime/detector/test_detector_types.py` + Gate: review + Scope: mark **both** `detect_info` and `detect_usage` `@abstractmethod`, so a future vendor + cannot silently skip the split — `raise NotImplementedError` would only fail when called, which + is exactly the silence being removed, and a vendor with genuinely no usage query is better off + declaring a two-line stub than inheriting a no-op that makes `detect()` look like it measured + something. All nine already implement both, so the cost is zero. Also migrate the two + expand-state fixtures in `test_detector_types.py` (`_UnimplementedDetector`, `_InfoOnlyDetector`) + that exist precisely to pin the un-contracted state; delete `get_physical_function_by_bdf` if no caller + remains; add `detect --no-usage`, rendering `N/A` instead of `0` in the table for the omitted + columns; ~~write the per-vendor parity table (driver calls made, and every deliberate divergence + with its reason) into `docs/`~~. + **The parity document was withdrawn after the build, by request** — `docs/detector-parity.md`, + its `mkdocs.yml` nav entry and the test asserting every vendor appeared in it were all removed, + so `docs/` and `mkdocs.yml` carry nothing from this work. Nothing it recorded is lost: the + deferred items it carried are the **Deferred** table below, and every divergence it explained + is stated in a comment at the code that makes it. + Acceptance: a `Detector` subclass without `detect_info` fails to instantiate; `detect --no-usage` + prints `N/A` for utilization/temperature and issues no metric call; ~~the parity table names all + nine vendors and every divergence recorded in F1~~ (withdrawn with the document); + **no detector can emit a `vgpu` key** — + each vendor task could only assert its own output, so this guard lands here, scoped to the + shipped package (`gpustack_runtime/**/*.py`). Deliberately **not** a literal tree-wide text + grep: `test_ascend.py` plants a stale `vgpu` in an appendix on purpose, to prove it cannot + resurrect `/dev/vdavinci`, and the captured `samples/` are covered by `T12`'s own schema guard. + Verify: `uv run pytest tests/gpustack_runtime/detector -q && make docs` + +- [x] **T12 · Align `samples/detect_output_*.json` with the new Device contract** + Blocked by: T3, T4, T5, T6, T7, T8, T9, T10 + Owns: `tests/gpustack_runtime/detector/samples/**`, + `tests/gpustack_runtime/detector/test_samples.py` + Scope: realign every `detect_output_*.json` field-by-field to the post-change shape — no `vgpu`; + MIG instances inside the card's `appendix["mig_devices"]` (`detect_output_nvidia_h100_mig.json` + still uses the retired standalone-MIG shape with `gpu_instance_index` / + `compute_instance_index`); the physical-index appendix keys present; NVIDIA totals left as the + driver reports them. Add the schema-guard test that loads every sample and deserializes it, + so the samples stop drifting silently. `topology_output_*.json` is untouched (topology is a + Non-Goal). + Acceptance: every `detect_output_*.json` deserializes into `Device`; none carries `vgpu`, or + `gpu_instance_index` / `compute_instance_index` at the top level; MIG samples carry + `appendix["mig_devices"]`; the guard test fails when a sample grows a key `Device` cannot hold. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_samples.py -q` + +#### Deployer track + +- [x] **D1 · Pin the device ordering for containers that see every card** + Blocked by: None + Owns: `gpustack_runtime/deployer/__types__.py`, `gpustack_runtime/deployer/docker.py`, + `gpustack_runtime/deployer/podman.py`, `gpustack_runtime/deployer/kuberentes.py`, + `tests/gpustack_runtime/deployer/test_visible_devices_ordering.py` + Gate: review + Scope: add `Deployer.map_visible_devices_ordering(runtime_envs) -> dict[str, str]` beside the + existing `map_backend_visible_devices` / `map_visible_devices_affinities` — it resolves each + runtime visible-devices env name to its manufacturer through `get_manufacturer()` and returns + `{"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}` when NVIDIA is among them, `{}` otherwise. Call it from all + three deployers' device-request loops (`docker.py`, `podman.py`, `kuberentes.py`), right beside + the existing `if r_v != "all" and privileged:` backend-visible-devices block, under the + complementary condition `r_v == "all" or privileged` — the deployers' own resolution of "this + container sees every card". Never overwrite a value the container already declares (the + Kubernetes site appends to a `V1EnvVar` list, so it needs a name check). + Acceptance: NVIDIA + `all` → the container carries `CUDA_DEVICE_ORDER=PCI_BUS_ID` on all three + deployers; NVIDIA + privileged with a specific device request → injected; NVIDIA + a specific + device request without privilege → not injected; a non-NVIDIA manufacturer → nothing injected; + a container declaring `CUDA_DEVICE_ORDER=FASTEST_FIRST` keeps its own value; the auto-mapping + resource key on a mixed-manufacturer node injects once, not once per manufacturer. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_visible_devices_ordering.py -q` + +- [x] **D2 · `WorkloadStatus` exit-status model** + Blocked by: D1 + Owns: `gpustack_runtime/deployer/__types__.py`, + `tests/gpustack_runtime/deployer/test_workload_status.py` + Gate: review + Scope: add a `WorkloadStatusExit` `dataclass_json` dataclass (name, token, exit_code, reason, + message, started_at, finished_at, restart_count) and a defaulted `WorkloadStatus` field holding a + list of them. Serial after D1 purely to keep a single writer on `__types__.py`. + Acceptance: a status with no exits round-trips; a payload serialized before this change still + deserializes; the new field is excluded from no existing consumer's expectations. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_workload_status.py -q` + +- [x] **D3 · Docker: populate exit status** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/docker.py`, + `tests/gpustack_runtime/deployer/test_docker_status.py` + Scope: fill the exit list from each container's `State` (`ExitCode`, `Error`, `OOMKilled`, + `StartedAt`, `FinishedAt`, `RestartCount`) for init and run containers, and carry the reason into + `state_message`. `parse_state`'s existing verdicts do not change. + Acceptance: exited with 137 and `OOMKilled` → the existing state verdict is preserved and an exit + entry carries 137 plus the `OOMKilled` reason; a running container contributes no entry; a + container whose `State` lacks the keys degrades to an entry with the code alone. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_docker_status.py -q` + +- [x] **D4 · Podman: populate exit status** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/podman.py`, + `tests/gpustack_runtime/deployer/test_podman_status.py` + Scope: the D3 change applied to the Podman deployer's own status class. Runs concurrently with D3 + and D5 — disjoint files. + Acceptance: same as D3, against Podman's container attrs. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_podman_status.py -q` + +- [x] **D5 · Kubernetes: exit status, image-pull failure, Pod Events, RBAC** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/kuberentes.py`, `deploy/manifests/kubernetes.yaml`, + `tests/gpustack_runtime/deployer/test_kubernetes_status.py` + Gate: review + Scope: fill the exit list from `container_statuses` / `init_container_statuses`, reading + `state.terminated` and `last_state.terminated` for codes and `state.waiting` for reason and + message; make `parse_state` return `Failed` — not `Pending` — for `ErrImagePull`, + `ImagePullBackOff`, `InvalidImageName` and `RegistryUnavailable`, with reason and message in + `state_message`; read Pod Events, field-selected to the Pod, only in that blocked state and + append the Event message; degrade a `403` to a debug log; grant `get`/`list` on core `events` in + the manifest. + Acceptance: Pending + `ImagePullBackOff` → `Failed` carrying the reason and the registry message; + `terminated(1, "Error")` → an exit entry with 1; Events raising `ApiException(403)` → state and + reason still reported and nothing raised; a Running Pod triggers no Events call. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_kubernetes_status.py -q` + +#### End-of-build review + +Three axes ran over the whole branch diff: the **spec** axis (`spec-reviewer`), the **standards** axis +(`agent-skills:code-reviewer`), and two external cross-checks (codex and kimi). The spec axis returned +Missing 0 / Unasked 0 / Wrong 1 — a self-contradiction of this document's own Boundaries on +`get_topology`, fixed at the source. The other three produced 20 findings between them, of which 14 +were real and are fixed above the C2 line, each with a regression test that fails without the fix: + +- **The Ascend `/dev/davinciN` fallback**, flagged High by codex and Critical by the standards axis + while kimi read it as preserved behaviour. All three were partly right: the pre-change default path + *did* build the node from the logic id, because the physical id was only used when + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` was set. So it is a pre-existing defect this work + preserved and can now close, not a regression it introduced — and it is exactly what this document's + "**Ask first:** before changing what a `/dev/...` device-node path resolves to" exists to catch. +- Two identity defects in the new usage join: an ambiguous UUID broadcasting one card's metrics onto + every card, and `cndevGetUUID` manufacturing a bare `"MLU-"` that is such an id. +- Two availability defects: a usage-query failure discarding an inventory already in hand, and every + Cambricon card being skipped reading as "no hardware". +- Two reporting defects a consumer would have acted on: `--format json` ignoring `--no-usage`, and a + container that merely exited non-zero carrying no reason and so no `state_message`. +- **Two tests that could not fail**, which matter more than the findings they hid: the usage-join test + used identical payloads for every card, so a positional join, a reversed mapping and a + broadcast-to-all all satisfied it; and the memory-status merge test ran with health checks off, where + both queries answer `healthy` without a driver call. + +#### Deferred — each needs its own task + +These were found during the build or its review, judged real, and deliberately **not** fixed here, +because each is a behaviour change wider than this work's scope. They were recorded in a parity +document that has since been dropped, so the list lives here: + +| Vendor | Deferred item | +| --- | --- | +| All | Eight bindings check `_libInitialized` **outside** the load lock, so two concurrent first callers can both reach the driver's init and the loser's error is cached as permanent state. `pycndev` is fixed; `pyamdsmi`, `pydcmi`, `pyhgml`, `pyixml`, `pymtml`, `pymxsml`, `pynvml` and `pyrocmsmi` are not — they keep upstream `pynvml`'s shape on purpose so they can be re-synced. One mechanical sweep, provable per binding with the same two-caller test. | +| Ascend | Utilization is read through the operator's **V1** path only, and only the AICORE rate; `pydcmi` binds no `_v2` utilization entry point at all. Needs a binding addition. | +| Ascend | `_get_device_memory_status` tries HBM ECC then DDR ECC unconditionally, where the operator picks the ECC device type from whichever memory query succeeded. Pre-existing. | +| Cambricon | The only vendor that degrades **per card**; the other eight lose the whole pass on one faulty card. Graceful degradation is very likely right everywhere, but that is a repo-wide change. | +| Cambricon | A card skipped by the **usage** pass keeps the information query's zeroes rather than reading as unmeasured. Its `memory_status` still carries the health that query read, so a failing card is not fully masked — but telling "not measured" from 0 needs `cores_utilization`, `memory_used` and `memory_utilization` to become optional, which is a cross-vendor change to `Device`, its consumers and every sample. | +| Cambricon | `cores` and the power **limit** are not reported: `pycndev` binds no core-count entry point and the operator reads neither. | +| MetaX | **The temperature divisor is unresolved** — see Open Questions. Needs hardware. | +| MThreads | `detect_info` frees the MTML handle in its `finally` and then uses it to open the memory context: a use-after-free. Pre-existing, and `detect_usage` does not reproduce it. | +| MThreads | `mtmlDeviceGetPowerUsage` is reported without unit conversion. Same follow-up as above. | + +- [x] **C1 · Checkpoint: whole suite, lint, CLI smoke** + Blocked by: T11, T12, D3, D4, D5 + Owns: None + Gate: review + Acceptance: `make lint` leaves the tree clean, `make test` is green, and both + `gpustack-runtime detect --format json` and `--no-usage --format json` run on the dev machine + returning an empty list without a traceback. + Verify: `make prepare && make lint && make test` + +- [x] **C2 · Checkpoint: hardware validation on the user-provided environments** + Blocked by: C1 + Owns: `tests/gpustack_runtime/detector/samples/**` + Gate: review + Scope: on each environment the user provides, run `detect` with and without `--no-usage`, compare + the reported card name, memory, cores and health against the operator's `Devices` object on the + same node — **health only with `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false`**, since the + default skips the ECC read entirely and would report `healthy` unconditionally (see F1's + all-vendors row), deploy one workload with an unpullable image to confirm the reported failure, and + refresh any sample whose real output differs from T12's hand-alignment. Also settles the Ascend + `/dev/davinci{N}` Open Question. + **Three value-level refreshes `T12` deliberately refused to invent, so they are `C2`'s to + capture:** the keys absent from the realigned samples (`minor_number` on NVIDIA and THead, + `physical_id` on Ascend, and the card-level fields of the two rebuilt MIG cards in + `detect_output_nvidia_h100_mig.json`) — their absence is honest, not a discrepancy; the two + `1c.3g.40gb` MIG instances, still carrying the *compute-instance* profile name from the capture + while today's detector reports the *GPU-instance* one; and the AMD / Hygon `name` values, which + now prefer the host's local pci.ids board name and so cannot be derived from a sample at all. + Acceptance: per environment, name/memory/health match the operator's report for every card; + `--no-usage` omits the metric fields; the unpullable-image workload reports `Failed` with the + pull reason; samples committed from real output. + + **Result.** All five environments ran `detect`, `detect --no-usage`, `topology` and a + `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false` pass, every one exiting 0 with an empty stderr: + 2 × AMD RX 7800 XT, 1 × RTX 5090 D, 2 × RTX 4090 48 GB, 8 × Ascend 910B2, 16 × T-Head + PPU-ZW810E. `--no-usage` omitted exactly the five usage keys and kept `memory`, `memory_status` + and `power`. Health read the same verdict with the ECC check on as off on every card, so no + card on these hosts is carrying an uncorrected error. + + Four things it measured that no fixture could: + + 1. **T-Head's minor number is not its enumeration index**, on any of the 16 cards. They ran one + apart here (index 3 → minor 4, index 15 → minor 16), and the operator's allocator records the + same relation on its own measured host — but **that offset is an observation about a host and + a driver, not a rule**, and nothing on either side may compute one number from the other. The + operator says so at `allocator/thead/mig.go:640-645`, and its own note explains the offset on + that host: `/dev/alixpu` holds minor 0 of the same character-device major, so the per-card + nodes start one along. What the measurement establishes is only the load-bearing part — the + two numbers differ — which is why `/dev/alixpu_ppu` is named by the **card ordinal** and + the minor is read from `hgmlDeviceGetMinorNumber` as the card's *identity*. Using the + identity as the name lands on the neighbouring card, and on the last card of a 16-card host + names a `ppu16` that does not exist at all. + 2. **Ascend reports `physical_id` for all 8 NPUs**, so the invariant the Ascend fix relies on + holds on a real driver. Logic and physical id agree here (0–7), which is why the fix is also + provably non-regressive on this host. + 3. **AMD's per-card UUIDs are distinct** (`GPU-5c88007d760374f3` / `GPU-d99e7fe92c7bdf75`), so + `rsmi_dev_unique_id_get` answering a constant is not happening here. The merge's + ambiguous-id guard stays as defence in depth; no identity change is needed. + 4. **Hygon by proxy executes clean** on the AMD host. Its appendix lacks `card_id` / + `renderd_id` there, which is *not* a defect: it reads `/sys/module/hycu` and + `/sys/module/hydcu`, and the host runs `amdgpu`. `driver_version` is null for the same + reason. Exactly the "cannot prove a Hygon-specific value" limit recorded above. The vendor-id + edit was reverted and never committed. + + `F5` end to end, on the k3s cluster of the RTX 5090 host and on Docker 29.6.1 of the AMD host: + an unpullable image reports `state = Failed` with + `state_message = "ImagePullBackOff: Back-off pulling image ...; Error: ErrImagePull"` — the + waiting reason *plus* the appended Pod Event — and an `exits` entry whose `started_at` and + `finished_at` are empty because it never ran. A container exiting 7 reports `exit_code = 7`, + `reason = "Error"` and `state_message = "Error"`, which is the end-of-build fix: before it, + Docker left `State.Error` empty for a container that ran and exited non-zero, so the commonest + crash of all carried no reason and set no message. Its timestamps came back with six-digit + microseconds, i.e. the nanosecond truncation holds against a real daemon. + + **What C2 could not settle**, as scoped: Iluvatar, MetaX, MThreads and Cambricon have no + environment, so the MetaX temperature divisor and Cambricon's MB-means-MiB reading keep their + markers; and no MIG-enabled host existed, so `detect_output_nvidia_h100_mig.json`'s two rebuilt + cards and their compute-instance profile names stay unrefreshed. Both are recorded in the + samples README rather than guessed at. + Verify: `uv run gpustack-runtime detect --format json` on the provided host (access method + supplied by the user at that point) + +### Test Plan + +[x] I/we understand the owners of the involved components may require updates to existing tests to make +this code solid enough prior to committing the changes necessary to implement this enhancement. + +#### Prerequisite testing updates + +- `tests/gpustack_runtime/detector/fixtures/` holds only a `load()` helper and no data files. Every + detector test today is `skipif not is_supported()` and therefore never runs in CI, so each vendor + task must bring a fake binding — fake device handles plus a monkeypatch of its binding module — to + exercise `detect_info` / `detect_usage` on a machine with no driver. +- Each vendor task builds that fake **inside its own test module**, not in a shared + `fixtures/bindings.py`. The fakes have nothing to share in practice (`pynvml`, `pymxsml`, `pydcmi` + and the rest expose unrelated APIs), while one shared file would serialize seven vendor tasks whose + paths are otherwise disjoint — the coordination cost outweighs the deduplication. +- Every fake carries a **call log**, because several acceptance criteria are "issues **no** metric + call" and cannot be asserted from return values. A dozen lines per vendor. +- `pytest.ini` lists an `integration_tests` testpath that does not exist — create the directory (see + Integration tests) or drop the entry. +- Fixture `pci.ids` extract for `get_pci_device_name()` (T1) covering one AMD and one Hygon + vendor:device pair. +- Optional: add `pytest-cov` so the per-package numbers below become measurable rather than reviewed. + +#### Unit tests + +Coverage is not measured today (`pytest.ini`'s `addopts` carries no `--cov`), so these are entry targets +for the non-hardware code paths, not deltas from a baseline: + +- `gpustack_runtime/detector`: `2026-08-14` - not measured; target `>=70%` +- `gpustack_runtime/detector/pycndev`: `2026-08-14` - not measured (new); target `>=60%` (struct + layouts and error mapping; the library-loaded path stays hardware-gated) +- `gpustack_runtime/deployer` (status parsing and plan defaulting only): `2026-08-14` - not measured; + target `>=60%` + +Per-unit coverage added by task: the ABC composition and usage merge (T1); `cndev` struct layouts and +error mapping (T2); each vendor's `detect_info` / `detect_usage` split, its parity fix and the absence +of `vgpu` (T3–T10); abstractness of `detect_info` and the `--no-usage` rendering (T11); the sample +schema guard (T12); the ordering injection across all/privileged/subset/non-NVIDIA/already-declared +and all three deployers (D1); exit-status +round-tripping and backward-compatible deserialization (D2); Docker, Podman and Kubernetes exit-status +parsing plus the image-pull verdict and the Events `403` degradation (D3–D5). + +#### Integration tests + +Needing a real Docker daemon and a real Kubernetes cluster, so they live under `integration_tests/` and +are not part of `make test`'s default run. Concrete test names are added after the implementation PR +merges: + +- Docker: a workload whose run container exits non-zero reports the exit code and reason. +- Docker: a workload killed by the OOM killer reports `OOMKilled`. +- Kubernetes: a workload with an unresolvable image (`does-not-exist.invalid/x:y`) reports `Failed` + with the pull reason and the Event message. +- Kubernetes: the same workload with the `events` RBAC rule removed still reports `Failed` with the + reason, and logs rather than raises. +- Kubernetes: a healthy running workload triggers no Events request (asserted on the API call log). + +#### e2e tests + +None automated in this repository. `gpustack_runtime` is a library consumed by the GPUStack worker and +the operator, and end-to-end coverage belongs to those repositories' suites; duplicating it here would +require provisioning nine vendors' hardware in CI. The end-to-end guarantee for this change is +checkpoint `C2`: a manual pass on the environments the user provides, comparing the runtime's report +against the operator's `Devices` object on the same node and deploying one unpullable-image workload. + +## Alternatives + +- **Keep a single `detect()` with a boolean flag, no split.** Rejected: a caller that only wants a + metrics refresh would still re-run the whole inventory query, and the operator's proven + detect/monitor separation would have no counterpart. +- **Mirror the operator's names (`detect_accelerator` / `monitor_accelerator`).** Rejected: the + runtime's public surface is `detect_devices` / `Detector.detect`; renaming churns every caller for + cosmetic symmetry. +- **Copy `CUDA_DEVICE_ORDER` from the worker process's environment.** Rejected: it only works if + something upstream remembers to set it on the worker, which is the same configuration gap that + produced issue #6041 in the first place. Whether a container sees every card is something the + deployer knows and the worker does not, so the deployer decides. **Confirmed independently by the + operator**, which reached the same conclusion on its own side and declined to set the variable on + the worker Deployment — "that process runs no CUDA and propagates nothing to workloads, so the + ordering has to be stated where the injection is consumed" (`fix(devicemanager): pin CUDA's device + order where a slice's limits are read by ordinal`). Copying from the worker would therefore have + been copying a variable nobody sets. +- **Inject `CUDA_DEVICE_ORDER=PCI_BUS_ID` into every NVIDIA workload unconditionally.** Rejected: a + container given a specific device subset already has a scoped visible-devices env, and pinning the + ordering there would silently change the ordinals a caller addressing that subset relies on. +- **Introduce a generic `GPUSTACK_RUNTIME_DEPLOY_INHERIT_ENVS` passthrough list.** Rejected for now: + a new configuration surface for a single known variable. Revisit when a second one appears. +- **Report image-pull failures from `container_statuses` only, without Pod Events.** Rejected: the + waiting reason alone ("ImagePullBackOff") omits the registry error the user needs; Events carry it. +- **Keep the `vgpu` flag but stop acting on it.** Rejected: a field nothing consumes is drift waiting + to happen, and the Ascend CDI path proves consumers appear. + +## Open Questions + +- ~~Does `/dev/davinci{N}` follow the DCMI logic id or the physical id on the driver versions we + support?~~ **Answered by the end-of-build review: the physical id, and nothing stands in for it.** + The operator's `ascend/device.go` skips a device whose `GetPhysicalID` fails rather than guessing, + and so does the detector now; the CDI generator refuses to build a path without + `appendix["physical_id"]`. `C2` no longer decides this — it confirms it, by checking that the + refreshed Ascend samples carry a `physical_id` and that the node paths address the right NPUs. +- **MetaX temperature is scaled three different ways and at most one can be right.** The runtime + divides `mxSmlGetTemperatureInfo(_, TEMPERATURE_HOTSPOT)` by 100 with the comment `mC to C` — but + millidegrees to degrees is a division by 1000, and the very next line converts board power with + `// 1000 # mW to W`, so the two are inconsistent with each other. The operator applies **no** + conversion at all (`temperature = uint32(tempInfo)`). So the runtime reports the driver value ÷100 + and the operator reports it raw, which is exactly the Story 1 class of discrepancy this work exists + to remove. Found while building `T8`; deliberately left untouched, because `F1`'s MetaX row scopes + that task to the PF skip and guessing at a unit without hardware would just move the error. + **Settled by `C2`**: read one card's hotspot temperature on a real MetaX host and compare all three + readings against `mx-smi`. +- Should the usage query also be exposed at module level (e.g. `monitor_devices(devices)`) for + callers running their own polling loop, or is `Detector.detect_usage` enough? The plan ships only + `detect_devices(..., usage=...)`; add the module-level entry point when a caller needs it. +- Does the pinned `nvidia-ml-py>=13.580.65` expose `nvmlDeviceGetMemoryInfo(handle, version=2)` and + a v2 PCI-info accessor under those names, or is a version probe needed? **Answered while building + `T5`** — if the names differ, `T5` adds the probe rather than pinning a newer floor. +- ~~Is there an upstream Python `cndev` binding worth depending on instead of hand-writing + `pycndev`?~~ **Answered by `T2`: no.** The name does not exist on PyPI (checked against the full + simple index), Cambricon publishes no Python device-management binding at all — their own + `libcndev.so` consumers (`mlu-exporter`, `cambricon-k8s-device-plugin`) are Go, and `mlu-exporter` + makes the operator supply `cndev.h` themselves because the header ships with the driver. The only + third-party candidate, `mlu-api`, vendors its own `libcndev.so` (so it cannot track the host + driver's ABI), is cp310/x86_64-only and exposes device count plus core utilization and nothing + else. CNDev is a driver-versioned C ABI, i.e. the `pydcmi`/`pymxsml` situation, not the + `pynvml`/`pymtml` one — hand-written it is. diff --git a/tests/gpustack_runtime/detector/samples/README.md b/tests/gpustack_runtime/detector/samples/README.md index ed5c025..4a24e37 100644 --- a/tests/gpustack_runtime/detector/samples/README.md +++ b/tests/gpustack_runtime/detector/samples/README.md @@ -1,3 +1,46 @@ # GPUStack Runtime Detector Samples This directory contains output samples fetched by the GPUStack Runtime Detector. + +`test_samples.py` guards the shape of every `detect_output_*.json` against the `Device` contract. + +## Captured from real hardware + +These were captured with `gpustack-runtime detect --format json` / `topology --format json` on the +host named, so their values are measurements rather than hand-alignments: + +| Sample | Host | +| --- | --- | +| `*_amd_rx7800xt.json` | 2 × AMD Radeon RX 7800 XT, ROCm 7.2.0, driver 6.16.13 | +| `*_ascend_910b2.json` | 8 × Ascend 910B2 | +| `*_nvidia_rtx4090_48g.json` | 2 × RTX 4090 48 GB, driver 595.84 | +| `*_nvidia_rtx5090d.json` | 1 × RTX 5090 D, driver 595.84 | +| `*_thead_ppu.json` | 16 × PPU-ZW810E | + +Two facts worth reading off them, because both decide a device-node path: + +- **Ascend carries `appendix["physical_id"]` on every device.** `/dev/davinciN` is numbered by it, and + a device whose physical id cannot be read is dropped rather than addressed by the logic id. On this + host the two happen to agree (0–7). +- **T-Head's `appendix["minor_number"]` is not its enumeration index.** They run one apart on this host + (index 3 → minor 4, index 15 → minor 16) because `/dev/alixpu` holds minor 0 of the same + character-device major, but that offset is an observation about this host and driver, not a rule — + neither number is ever computed from the other. `/dev/alixpu_ppu` is named by the *card ordinal*; + the minor is the card's identity, and the CDI generator compares it against the node's own minor to + prove the ordinal reached the right card. + +## Not captured from real hardware + +The remaining samples are hand-aligned and carry the caveats below. + +In `detect_output_nvidia_h100_mig.json`, the two MIG-enabled cards (index 0 and 3) are rebuilt by +hand from the instances that the retired standalone-MIG shape reported in their place, so they carry +only the fields those instances copied off the card: `name`, `uuid`, `cores`, `memory`, the usage +fields and the fabric keys are absent because the old shape did not record them, not because the +driver had no answer. Card 3's instances are still named after their compute-instance profile +(`1c.3g.40gb`), where the detector now reports the GPU-instance profile. Refresh from real hardware +before relying on either — no MIG-enabled host was available. + +`detect_output_hygon_k100ai.json` is likewise unrefreshed. The Hygon *code path* was exercised on the +AMD host by pointing its PCI vendor gate at `0x1002`, which proves the path runs and its field names +resolve, but the values it produced are an AMD card's and were deliberately not written here. diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json index bf37d61..2e4c822 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json @@ -19,7 +19,6 @@ "power_used": 132, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:26:00.0", "numa": "0", "card_id": 9, @@ -46,7 +45,6 @@ "power_used": 141, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:46:00.0", "numa": "0", "card_id": 17, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json index d0f1132..777c301 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json @@ -19,7 +19,6 @@ "power_used": 166, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:25:00.0", "numa": "0", "card_id": 0, @@ -46,7 +45,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:2a:00.0", "numa": "0", "card_id": 8, @@ -73,7 +71,6 @@ "power_used": 162, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:45:00.0", "numa": "0", "card_id": 16, @@ -100,7 +97,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:4a:00.0", "numa": "0", "card_id": 24, @@ -127,7 +123,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:a5:00.0", "numa": "1", "card_id": 32, @@ -154,7 +149,6 @@ "power_used": 162, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:a8:00.0", "numa": "1", "card_id": 40, @@ -181,7 +175,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:c5:00.0", "numa": "1", "card_id": 48, @@ -208,7 +201,6 @@ "power_used": 158, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:c8:00.0", "numa": "1", "card_id": 56, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json index 6207deb..f41cda4 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json @@ -2,28 +2,53 @@ { "manufacturer": "amd", "index": 0, - "name": "Navi 32 [Radeon RX 7700 XT / 7800 XT]", + "name": "AMD Radeon RX 7800 XT", "uuid": "GPU-5c88007d760374f3", - "driver_version": "6.12.12", - "runtime_version": "7.1", - "runtime_version_original": "7.1.1", + "driver_version": "6.16.13", + "runtime_version": "7.2", + "runtime_version_original": "7.2.0", "compute_capability": "gfx1101", "cores": 60, - "cores_utilization": 19, + "cores_utilization": 13, "memory": 16368, - "memory_used": 206, - "memory_utilization": 1.26, + "memory_used": 174, + "memory_utilization": 1.06, "memory_status": "healthy", - "temperature": 34, + "temperature": 43, "power": 236, - "power_used": 17, + "power_used": 10, "appendix": { "arch_family": "GC 11.0.0", - "vgpu": false, - "bdf": "0000:03:00.0", + "bdf": "0000:04:00.0", "numa": "0", "card_id": 1, "renderd_id": 128 } + }, + { + "manufacturer": "amd", + "index": 1, + "name": "AMD Radeon RX 7800 XT", + "uuid": "GPU-d99e7fe92c7bdf75", + "driver_version": "6.16.13", + "runtime_version": "7.2", + "runtime_version_original": "7.2.0", + "compute_capability": "gfx1101", + "cores": 60, + "cores_utilization": 34, + "memory": 16368, + "memory_used": 174, + "memory_utilization": 1.06, + "memory_status": "healthy", + "temperature": 44, + "power": 212, + "power_used": 10, + "appendix": { + "arch_family": "GC 11.0.0", + "bdf": "0000:07:00.0", + "numa": "0", + "card_id": 0, + "renderd_id": 129 + } } ] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json b/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json index b6d0254..5e7291c 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json @@ -19,7 +19,6 @@ "power_used": null, "appendix": { "arch_family": "Ascend310P3", - "vgpu": false, "bdf": "0000:81:00.0", "numa": "2", "card_id": 4, @@ -47,7 +46,6 @@ "power_used": null, "appendix": { "arch_family": "Ascend310P3", - "vgpu": false, "bdf": "0000:81:00.0", "numa": "2", "card_id": 4, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json b/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json index d433463..85a680f 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json @@ -3,248 +3,248 @@ "manufacturer": "ascend", "index": 0, "name": "910B2", - "uuid": "7281A664 C0C977 2879FD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "E0F4EE64 802061B1 6A691492 89528485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3421, - "memory_utilization": 5.22, + "memory_used": 3432, + "memory_utilization": 5.24, "memory_status": "healthy", - "temperature": 49, + "temperature": 47, "power": null, - "power_used": 99.9, + "power_used": 101.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:c1:00.0", - "numa": "6", "card_id": 0, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.6", + "physical_id": 0, + "numa": "6", + "roce_ip": "10.52.32.11", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 1, "name": "910B2", - "uuid": "C0DBA664 C107DB 2A9F372 DFD00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "E281A66C 140C979 2CFBED72 A4500485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 12458, - "memory_utilization": 19.01, + "memory_used": 3427, + "memory_utilization": 5.23, "memory_status": "healthy", - "temperature": 51, + "temperature": 45, "power": null, - "power_used": 100.0, + "power_used": 101.0, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:01:00.0", - "numa": "0", "card_id": 1, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.5", + "physical_id": 1, + "numa": "0", + "roce_ip": "10.52.32.10", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 2, "name": "910B2", - "uuid": "6EC1A664 100D35B 6BB27372 DFD00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "7281A664 1003374 3444AB72 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3413, - "memory_utilization": 5.21, + "memory_used": 3420, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 49, + "temperature": 45, "power": null, - "power_used": 100.2, + "power_used": 96.2, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:c2:00.0", - "numa": "6", "card_id": 2, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.8", + "physical_id": 2, + "numa": "6", + "roce_ip": "10.52.32.13", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 3, "name": "910B2", - "uuid": "6281A66C 80F7B1 11DAB172 99D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "687DA66C 100D2DF 1915AB72 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3412, - "memory_utilization": 5.21, + "memory_used": 3421, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 100.3, + "power_used": 97.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:02:00.0", - "numa": "0", "card_id": 3, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.7", + "physical_id": 3, + "numa": "0", + "roce_ip": "10.52.32.12", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 4, "name": "910B2", - "uuid": "6281A66C 120F0F3 1BD3EB72 86D28485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "D0DBA664 12066DD 27A93572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3410, - "memory_utilization": 5.2, + "memory_used": 3421, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 49, + "temperature": 44, "power": null, - "power_used": 102.2, + "power_used": 95.0, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:81:00.0", - "numa": "4", "card_id": 4, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.11", + "physical_id": 4, + "numa": "4", + "roce_ip": "10.52.32.16", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 5, "name": "910B2", - "uuid": "1E82A664 100D179 346CBD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "3281A664 1404B1E 1AC3572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3408, - "memory_utilization": 5.2, + "memory_used": 3420, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 104.5, + "power_used": 102.6, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:41:00.0", - "numa": "2", "card_id": 5, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.12", + "physical_id": 5, + "numa": "2", + "roce_ip": "10.52.32.17", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 6, "name": "910B2", - "uuid": "3EC1A664 60C27A 1902B172 99D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "7281A664 1207247 20B9E972 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3408, - "memory_utilization": 5.2, + "memory_used": 3419, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 45, "power": null, - "power_used": 116.6, + "power_used": 102.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:82:00.0", - "numa": "4", "card_id": 6, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.10", + "physical_id": 6, + "numa": "4", + "roce_ip": "10.52.32.15", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 7, "name": "910B2", - "uuid": "7281A664 E0C977 690ABD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "785BA664 C0D378 3411B572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3409, - "memory_utilization": 5.2, + "memory_used": 3426, + "memory_utilization": 5.23, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 102.2, + "power_used": 99.9, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:42:00.0", - "numa": "2", "card_id": 7, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.9", + "physical_id": 7, + "numa": "2", + "roce_ip": "10.52.32.14", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } } ] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json b/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json index 88a606f..41545ca 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json @@ -18,7 +18,6 @@ "power": 400, "power_used": 104, "appendix": { - "vgpu": false, "bdf": "0000:07:00.0", "numa": "0", "card_id": 1, @@ -44,7 +43,6 @@ "power": 400, "power_used": 107, "appendix": { - "vgpu": false, "bdf": "0000:0a:00.0", "numa": "0", "card_id": 2, @@ -70,7 +68,6 @@ "power": 400, "power_used": 106, "appendix": { - "vgpu": false, "bdf": "0000:0f:00.0", "numa": "0", "card_id": 3, @@ -96,7 +93,6 @@ "power": 400, "power_used": 105, "appendix": { - "vgpu": false, "bdf": "0000:16:00.0", "numa": "0", "card_id": 4, @@ -122,7 +118,6 @@ "power": 400, "power_used": 108, "appendix": { - "vgpu": false, "bdf": "0000:1d:00.0", "numa": "0", "card_id": 5, @@ -148,7 +143,6 @@ "power": 400, "power_used": 110, "appendix": { - "vgpu": false, "bdf": "0000:20:00.0", "numa": "0", "card_id": 6, @@ -174,7 +168,6 @@ "power": 400, "power_used": 110, "appendix": { - "vgpu": false, "bdf": "0000:25:00.0", "numa": "0", "card_id": 7, @@ -200,7 +193,6 @@ "power": 400, "power_used": 108, "appendix": { - "vgpu": false, "bdf": "0000:2c:00.0", "numa": "0", "card_id": 8, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json b/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json index 8b64535..69d1b55 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json @@ -18,7 +18,6 @@ "power": 350, "power_used": 71, "appendix": { - "vgpu": false, "bdf": "0000:0e:00.0", "numa": "0", "card_id": 1, @@ -44,7 +43,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:0f:00.0", "numa": "0", "card_id": 2, @@ -70,7 +68,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:10:00.0", "numa": "0", "card_id": 3, @@ -96,7 +93,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:12:00.0", "numa": "0", "card_id": 4, @@ -122,7 +118,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:35:00.0", "numa": "0", "card_id": 5, @@ -148,7 +143,6 @@ "power": 350, "power_used": 58, "appendix": { - "vgpu": false, "bdf": "0000:36:00.0", "numa": "0", "card_id": 6, @@ -174,7 +168,6 @@ "power": 350, "power_used": 56, "appendix": { - "vgpu": false, "bdf": "0000:37:00.0", "numa": "0", "card_id": 7, @@ -200,7 +193,6 @@ "power": 350, "power_used": 56, "appendix": { - "vgpu": false, "bdf": "0000:38:00.0", "numa": "0", "card_id": 8, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json index 89734b2..ffb2d4c 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json @@ -19,7 +19,7 @@ "power_used": 12, "appendix": { "arch_family": "Blackwell", - "vgpu": false, + "mig": false, "bdf": "0000:ca:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json index 2d66611..aa08aa9 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json @@ -19,7 +19,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:0e:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -46,7 +46,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:15:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -73,7 +73,7 @@ "power_used": 67, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:1b:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -100,7 +100,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:67:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -127,7 +127,7 @@ "power_used": 69, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:9a:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -154,7 +154,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ab:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -181,7 +181,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ba:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -208,7 +208,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:db:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json index f451245..54d3464 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json @@ -1,92 +1,102 @@ [ { "manufacturer": "nvidia", - "index": 8, - "name": "MIG 1g.20gb", - "uuid": "MIG-667481b3-d832-5bc3-81ee-2f27a4f6cdf2", + "index": 0, "driver_version": "575.57.08", "runtime_version": "12.9", "runtime_version_original": "12.9.0", "compute_capability": "9.0", - "cores": 26, - "cores_utilization": 0, - "memory": 20096, - "memory_used": 16, - "memory_utilization": 0.08, - "memory_status": "healthy", - "temperature": 43, - "power": 700, - "power_used": 71, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 6, - "compute_instance_id": 0, - "gpu_instance_index": 57, - "compute_instance_index": 58 - } - }, - { - "manufacturer": "nvidia", - "index": 9, - "name": "MIG 1g.10gb", - "uuid": "MIG-30be8fbd-3a0a-59e7-ab47-88215d1c7022", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 9984, - "memory_used": 16, - "memory_utilization": 0.16, - "memory_status": "healthy", - "temperature": 43, - "power": 700, - "power_used": 71, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 11, - "compute_instance_id": 0, - "gpu_instance_index": 102, - "compute_instance_index": 103 - } - }, - { - "manufacturer": "nvidia", - "index": 10, - "name": "MIG 1g.10gb", - "uuid": "MIG-2808b45e-8825-5a89-8210-4f096cb502fe", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 9984, - "memory_used": 16, - "memory_utilization": 0.16, - "memory_status": "healthy", "temperature": 43, "power": 700, "power_used": 71, "appendix": { "arch_family": "Hopper", - "vgpu": true, - "sliced": true, + "mig": true, "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 12, - "compute_instance_id": 0, - "gpu_instance_index": 111, - "compute_instance_index": 112 + "mig_devices": [ + { + "index": 8, + "name": "1g.20gb", + "uuid": "MIG-667481b3-d832-5bc3-81ee-2f27a4f6cdf2", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 26, + "cores_utilization": 0, + "memory": 20096, + "memory_used": 16, + "memory_utilization": 0.08, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 6, + "compute_instance_id": 0 + } + }, + { + "index": 9, + "name": "1g.10gb", + "uuid": "MIG-30be8fbd-3a0a-59e7-ab47-88215d1c7022", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 9984, + "memory_used": 16, + "memory_utilization": 0.16, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 11, + "compute_instance_id": 0 + } + }, + { + "index": 10, + "name": "1g.10gb", + "uuid": "MIG-2808b45e-8825-5a89-8210-4f096cb502fe", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 9984, + "memory_used": 16, + "memory_utilization": 0.16, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 12, + "compute_instance_id": 0 + } + } + ], + "numa": "0-1" } }, { @@ -109,7 +119,7 @@ "power_used": 73, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0b.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -136,7 +146,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0c.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -145,62 +155,75 @@ }, { "manufacturer": "nvidia", - "index": 32, - "name": "MIG 1c.3g.40gb", - "uuid": "MIG-a72639ea-72c4-5068-adaa-30ed6ec08a86", + "index": 3, "driver_version": "575.57.08", "runtime_version": "12.9", "runtime_version_original": "12.9.0", "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 40448, - "memory_used": 46, - "memory_utilization": 0.11, - "memory_status": "healthy", "temperature": 41, "power": 700, "power_used": 73, "appendix": { "arch_family": "Hopper", - "vgpu": true, - "sliced": true, + "mig": true, "bdf": "0000:00:0d.0", - "numa": "0-1", - "gpu_instance_id": 2, - "compute_instance_id": 0, - "gpu_instance_index": 426, - "compute_instance_index": 427 - } - }, - { - "manufacturer": "nvidia", - "index": 33, - "name": "MIG 1c.3g.40gb", - "uuid": "MIG-cc07225c-ed33-5047-aff0-83a245a0c73a", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 26, - "cores_utilization": 0, - "memory": 40448, - "memory_used": 46, - "memory_utilization": 0.11, - "memory_status": "healthy", - "temperature": 41, - "power": 700, - "power_used": 73, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0d.0", - "numa": "0-1", - "gpu_instance_id": 2, - "compute_instance_id": 1, - "gpu_instance_index": 426, - "compute_instance_index": 428 + "mig_devices": [ + { + "index": 32, + "name": "1c.3g.40gb", + "uuid": "MIG-a72639ea-72c4-5068-adaa-30ed6ec08a86", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 40448, + "memory_used": 46, + "memory_utilization": 0.11, + "memory_status": "healthy", + "temperature": 41, + "power": 700, + "power_used": 73, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0d.0", + "numa": "0-1", + "gpu_instance_id": 2, + "compute_instance_id": 0 + } + }, + { + "index": 33, + "name": "1c.3g.40gb", + "uuid": "MIG-cc07225c-ed33-5047-aff0-83a245a0c73a", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 26, + "cores_utilization": 0, + "memory": 40448, + "memory_used": 46, + "memory_utilization": 0.11, + "memory_status": "healthy", + "temperature": 41, + "power": 700, + "power_used": 73, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0d.0", + "numa": "0-1", + "gpu_instance_id": 2, + "compute_instance_id": 1 + } + } + ], + "numa": "0-1" } }, { @@ -223,7 +246,7 @@ "power_used": 74, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0e.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -250,7 +273,7 @@ "power_used": 72, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0f.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -277,7 +300,7 @@ "power_used": 78, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:10.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -304,7 +327,7 @@ "power_used": 71, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:11.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json index e74ed42..b2c2c26 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json @@ -19,7 +19,7 @@ "power_used": 685, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:18:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -46,7 +46,7 @@ "power_used": 701, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:29:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -73,7 +73,7 @@ "power_used": 699, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:3a:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -100,7 +100,7 @@ "power_used": 689, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:4b:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -127,7 +127,7 @@ "power_used": 683, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:9a:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -154,7 +154,7 @@ "power_used": 698, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:aa:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -181,7 +181,7 @@ "power_used": 693, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ba:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -208,7 +208,7 @@ "power_used": 692, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ca:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json index 222b95a..d0cc879 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json @@ -19,7 +19,7 @@ "power_used": 11, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:01:00.0", "numa": "0" } @@ -44,7 +44,7 @@ "power_used": 5, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:72:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json new file mode 100644 index 0000000..1c8eb5f --- /dev/null +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json @@ -0,0 +1,54 @@ +[ + { + "manufacturer": "nvidia", + "index": 0, + "name": "NVIDIA GeForce RTX 4090", + "uuid": "GPU-bbc521eb-4a08-800b-eb2a-770a4a5b8a8c", + "driver_version": "610.43.02", + "runtime_version": "13.3", + "runtime_version_original": "13.3.0", + "compute_capability": "8.9", + "cores": 16384, + "cores_utilization": 0, + "memory": 49140, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 450, + "power_used": 29, + "appendix": { + "arch_family": "Ada-Lovelace", + "mig": false, + "bdf": "0000:02:00.0", + "minor_number": 0, + "numa": "0" + } + }, + { + "manufacturer": "nvidia", + "index": 1, + "name": "NVIDIA GeForce RTX 4090", + "uuid": "GPU-1068d7f6-26eb-5225-371e-58ca20150673", + "driver_version": "610.43.02", + "runtime_version": "13.3", + "runtime_version_original": "13.3.0", + "compute_capability": "8.9", + "cores": 16384, + "cores_utilization": 0, + "memory": 49140, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 450, + "power_used": 31, + "appendix": { + "arch_family": "Ada-Lovelace", + "mig": false, + "bdf": "0000:03:00.0", + "minor_number": 1, + "numa": "0" + } + } +] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json index a7b5a1e..8d8f022 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json @@ -19,7 +19,7 @@ "power_used": 10, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:04:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json index 101b449..9fd07c7 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json @@ -4,23 +4,24 @@ "index": 0, "name": "NVIDIA GeForce RTX 5090 D", "uuid": "GPU-31b58f27-ac45-ec52-c386-90aaeff2dfac", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", + "driver_version": "595.84", + "runtime_version": "13.2", + "runtime_version_original": "13.2.0", "compute_capability": "12.0", - "cores": 170, + "cores": 21760, "cores_utilization": 0, "memory": 32607, - "memory_used": 501, - "memory_utilization": 1.54, + "memory_used": 3, + "memory_utilization": 0.01, "memory_status": "healthy", - "temperature": 46, + "temperature": 51, "power": 575, "power_used": 21, "appendix": { "arch_family": "Blackwell", - "vgpu": false, + "mig": false, "bdf": "0000:01:00.0", + "minor_number": 0, "numa": "0" } } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json b/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json index c2da1a9..45e84eb 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json @@ -3,47 +3,399 @@ "manufacturer": "thead", "index": 0, "name": "PPU-ZW810E", - "uuid": "GPU-011ef111-8330-0426-0000-00006093bb5b", - "driver_version": "1.4.1-816bc0", - "runtime_version": "12.3", - "runtime_version_original": "12.3.0", + "uuid": "GPU-019e2226-4211-0208-0000-000000ab261d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, "compute_capability": "8.0", - "cores": 32768, + "cores": 16384, "cores_utilization": 0, - "memory": 97920, - "memory_used": 2, - "memory_utilization": 0.0, + "memory": 98304, + "memory_used": 91349, + "memory_utilization": 92.93, "memory_status": "healthy", - "temperature": 29, + "temperature": 34, "power": 400, - "power_used": 58, + "power_used": 93, "appendix": { - "vgpu": false, - "bdf": "0000:05:00.0", - "numa": "0" + "mig": false, + "bdf": "0000:a8:00.0", + "minor_number": 1, + "numa": "4" } }, { "manufacturer": "thead", "index": 1, "name": "PPU-ZW810E", - "uuid": "GPU-011ef111-8251-0426-0000-00006035ca38", - "driver_version": "1.4.1-816bc0", - "runtime_version": "12.3", - "runtime_version_original": "12.3.0", + "uuid": "GPU-019e2226-84c1-0200-0000-0000c020e701", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:a7:00.0", + "minor_number": 2, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 2, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-4422-0024-0000-000020178514", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:a9:00.0", + "minor_number": 3, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 3, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8261-0222-0000-0000a0490d60", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 31, + "power": 400, + "power_used": 94, + "appendix": { + "mig": false, + "bdf": "0000:aa:00.0", + "minor_number": 4, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 4, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-84b1-061e-0000-000020cc396a", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 33, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:65:00.0", + "minor_number": 5, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 5, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0571-0120-0000-0000405f5716", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:66:00.0", + "minor_number": 6, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 6, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8491-0424-0000-000020b1f22a", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 400, + "power_used": 95, + "appendix": { + "mig": false, + "bdf": "0000:64:00.0", + "minor_number": 7, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 7, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0571-031c-0000-0000e03d0464", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 400, + "power_used": 97, + "appendix": { + "mig": false, + "bdf": "0000:63:00.0", + "minor_number": 8, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 8, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8811-0618-0000-0000408ed50b", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:e8:00.0", + "minor_number": 9, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 9, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8491-0228-0000-000060ce486d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 94, + "appendix": { + "mig": false, + "bdf": "0000:e7:00.0", + "minor_number": 10, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 10, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8861-0430-0000-0000201ea851", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:e9:00.0", + "minor_number": 11, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 11, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8811-0420-0000-0000a01d582e", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 32, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:ea:00.0", + "minor_number": 12, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 12, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8851-021a-0000-000000e57d6f", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, "compute_capability": "8.0", - "cores": 32768, + "cores": 16384, "cores_utilization": 0, - "memory": 97920, - "memory_used": 2, + "memory": 98304, + "memory_used": 0, "memory_utilization": 0.0, "memory_status": "healthy", "temperature": 32, "power": 400, - "power_used": 59, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:25:00.0", + "minor_number": 13, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 13, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0581-0510-0000-00004031af7d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:26:00.0", + "minor_number": 14, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 14, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8591-061e-0000-000020791642", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 62, + "appendix": { + "mig": false, + "bdf": "0000:24:00.0", + "minor_number": 15, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 15, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8591-0510-0000-0000a066de79", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 64, "appendix": { - "vgpu": false, - "bdf": "0000:0b:00.0", + "mig": false, + "bdf": "0000:23:00.0", + "minor_number": 16, "numa": "0" } } diff --git a/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json b/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json index 8b1ae4e..8515803 100644 --- a/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json +++ b/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json @@ -3,16 +3,24 @@ "manufacturer": "amd", "devices_distances": [ [ + 0, + 50 + ], + [ + 50, 0 ] ], "devices_cpu_affinities": [ - "0-19" + "0-13", + "0-13" ], "devices_numa_affinities": [ + "0", "0" ], "appendices": [ + {}, {} ] } diff --git a/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json b/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json new file mode 100644 index 0000000..990d356 --- /dev/null +++ b/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json @@ -0,0 +1,27 @@ +[ + { + "manufacturer": "nvidia", + "devices_distances": [ + [ + 0, + 30 + ], + [ + 30, + 0 + ] + ], + "devices_cpu_affinities": [ + "0-13", + "0-13" + ], + "devices_numa_affinities": [ + "0", + "0" + ], + "appendices": [ + {}, + {} + ] + } +] diff --git a/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json b/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json index e0820ca..8f5e00b 100644 --- a/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json +++ b/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json @@ -4,31 +4,409 @@ "devices_distances": [ [ 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, 5 ], [ 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 10 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 10, 0 ] ], "devices_cpu_affinities": [ - "0-15", - "0-15" + "64-79,192-207", + "64-79,192-207", + "64-79,192-207", + "64-79,192-207", + "32-47,160-175", + "32-47,160-175", + "32-47,160-175", + "32-47,160-175", + "96-111,224-239", + "96-111,224-239", + "96-111,224-239", + "96-111,224-239", + "0-15,128-143", + "0-15,128-143", + "0-15,128-143", + "0-15,128-143" ], "devices_numa_affinities": [ + "4", + "4", + "4", + "4", + "2", + "2", + "2", + "2", + "6", + "6", + "6", + "6", + "0", + "0", "0", "0" ], "appendices": [ { "links_count": 7, - "links_state": 12, - "links_active_count": 2 + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 }, { "links_count": 7, - "links_state": 12, - "links_active_count": 2 + "links_state": 0, + "links_active_count": 0 } ] } diff --git a/tests/gpustack_runtime/detector/test_detector_cli.py b/tests/gpustack_runtime/detector/test_detector_cli.py new file mode 100644 index 0000000..c910ac4 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_detector_cli.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +from gpustack_runtime.cmds import detector as cmds_detector +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, +) + + +# --------------------------------------------------------------------------- # +# Helpers. # +# --------------------------------------------------------------------------- # +def _parse(*argv: str) -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + cmds_detector.DetectDevicesSubCommand.register(subparsers) + return parser.parse_args(["detect", *argv]) + + +def _device() -> Device: + return Device( + manufacturer=ManufacturerEnum.NVIDIA, + index=0, + name="NVIDIA A100-SXM4-40GB", + uuid="GPU-0", + driver_version="580.65.06", + runtime_version="13.0", + compute_capability="8.0", + cores=6912, + memory=40960, + memory_status=DeviceMemoryStatusEnum.HEALTHY, + appendix={}, + ) + + +# --------------------------------------------------------------------------- # +# `detect --no-usage` threading. # +# --------------------------------------------------------------------------- # +def test_detect_asks_for_usage_by_default(monkeypatch): + calls: list[dict] = [] + + def fake_detect_devices(**kwargs): + calls.append(kwargs) + return [] + + monkeypatch.setattr(cmds_detector, "detect_devices", fake_detect_devices) + + cmds_detector.DetectDevicesSubCommand(_parse("--format", "json")).run() + + assert calls == [{"fast": False, "usage": True}] + + +def test_detect_no_usage_skips_the_usage_query(monkeypatch): + calls: list[dict] = [] + + def fake_detect_devices(**kwargs): + calls.append(kwargs) + return [] + + monkeypatch.setattr(cmds_detector, "detect_devices", fake_detect_devices) + + cmds_detector.DetectDevicesSubCommand( + _parse("--no-usage", "--format", "json"), + ).run() + + # The flag is the only thing standing between the CLI and a metric call: + # every detector's usage query is skipped when `usage` is false. + assert calls == [{"fast": False, "usage": False}] + + +# --------------------------------------------------------------------------- # +# Table rendering: `N/A` for what was never measured. # +# --------------------------------------------------------------------------- # +def test_table_renders_measured_usage_as_numbers(): + dev = _device() + dev.cores_utilization = 42 + dev.memory_used = 1024 + dev.temperature = 61 + + table = cmds_detector.format_devices_table([dev]) + + assert "1024MiB / 40960MiB" in table + assert "42%" in table + assert "61C" in table + + +def test_table_renders_unmeasured_usage_as_not_available(): + table = cmds_detector.format_devices_table([_device()], usage=False) + + # A zero here is indistinguishable from a real idle reading, which is + # exactly the ambiguity `--no-usage` exists to remove. + assert "N/A / 40960MiB" in table + assert "0%" not in table + # The used-memory slot, i.e. everything left of the "/", carries no number. + assert not re.search(r"\d+MiB / ", table) + assert not re.search(r"\d+C", table) + # Information fields keep their values, and health is reported by both + # queries, so the status column stays meaningful. + assert "NVIDIA A100-SXM4-40GB" in table + assert "8.0" in table + assert "OK" in table + + +# --------------------------------------------------------------------------- # +# JSON rendering: an unmeasured field is absent, not zero. # +# --------------------------------------------------------------------------- # +def test_json_renders_measured_usage_as_numbers(): + dev = _device() + dev.cores_utilization = 42 + dev.memory_used = 1024 + dev.temperature = 61 + + payload = json.loads(cmds_detector.format_devices_json([dev])) + + assert payload[0]["cores_utilization"] == 42 + assert payload[0]["memory_used"] == 1024 + assert payload[0]["temperature"] == 61 + + +def test_json_omits_unmeasured_usage(): + dev = _device() + dev.appendix = {"mig_devices": [{"uuid": "MIG-0", "memory_used": 0}]} + + payload = json.loads(cmds_detector.format_devices_json([dev], usage=False)) + + # A serialized 0 is indistinguishable from a real idle reading, which is + # what `--no-usage` exists to remove -- and a machine-readable consumer is + # the more likely of the two to act on it. + for key in ("cores_utilization", "memory_used", "memory_utilization"): + assert key not in payload[0] + for key in ("temperature", "power_used"): + assert key not in payload[0] + # MIG instances carry the same fields, so they are dropped there too. + assert "memory_used" not in payload[0]["appendix"]["mig_devices"][0] + # Information fields keep their values, and health is reported by both + # queries, so the status stays meaningful. + assert payload[0]["memory"] == 40960 + assert payload[0]["memory_status"] == DeviceMemoryStatusEnum.HEALTHY + + +# --------------------------------------------------------------------------- # +# Tree-wide guard: no detector can emit a `vgpu` appendix key. # +# --------------------------------------------------------------------------- # +def test_no_detector_emits_a_vgpu_appendix_key(): + package = Path(cmds_detector.__file__).parent.parent + offenders = [ + str(path.relative_to(package)) + for path in sorted(package.rglob("*.py")) + if re.search(r"""['"]vgpu['"]""", path.read_text(encoding="utf-8")) + ] + + # The vGPU/virtual-card classification is gone for every vendor, so no + # shipped module may name the appendix key again. The vendor bindings' + # own `*_VGPU_*` constants are upstream SDK symbols, not appendix keys, + # and do not match. + assert offenders == [] diff --git a/tests/gpustack_runtime/detector/test_detector_types.py b/tests/gpustack_runtime/detector/test_detector_types.py new file mode 100644 index 0000000..febb5e0 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_detector_types.py @@ -0,0 +1,422 @@ +from __future__ import annotations + +import pytest + +from gpustack_runtime import envs +from gpustack_runtime.detector import ( + _DETECTORS, + Device, + DeviceMemoryStatusEnum, + Devices, + ManufacturerEnum, +) +from gpustack_runtime.detector.__types__ import ( + Detector, + merge_devices_usage, +) +from gpustack_runtime.detector.__utils__ import ( + _load_pci_device_names, + get_pci_device_name, +) + +# --------------------------------------------------------------------------- # +# Detector ABC: detect_info / detect_usage / detect composition. # +# --------------------------------------------------------------------------- # + + +class _RecordingDetector(Detector): + """ + A detector recording which query it was asked for, as the vendors behave + during the expand step: detect_info returns everything, detect_usage + merges a usage payload in. + """ + + def __init__( + self, + devices: Devices | None = None, + usages: Devices | None = None, + ): + super().__init__(ManufacturerEnum.UNKNOWN) + self.devices = devices + self.usages = usages + self.calls: list[str] = [] + + @staticmethod + def is_supported() -> bool: + return True + + def detect_info(self) -> Devices | None: + self.calls.append("info") + return self.devices + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + self.calls.append("usage") + if devices is None: + devices = self.detect_info() + return merge_devices_usage(devices, self.usages) + + +def _device(uuid: str, **kwargs) -> Device: + device = Device( + manufacturer=ManufacturerEnum.NVIDIA, + index=0, + name="NVIDIA A100-SXM4-40GB", + uuid=uuid, + cores=6912, + memory=40960, + appendix={}, + ) + for key, value in kwargs.items(): + setattr(device, key, value) + return device + + +def _usage(uuid: str, cores_utilization: int = 42) -> Device: + return Device( + uuid=uuid, + cores_utilization=cores_utilization, + memory_used=1024, + memory_utilization=2.5, + memory_status=DeviceMemoryStatusEnum.UNHEALTHY, + temperature=61, + power_used=250, + ) + + +def test_detect_composes_information_then_usage(): + det = _RecordingDetector(devices=[_device("GPU-0")], usages=[_usage("GPU-0")]) + + devices = det.detect() + + assert det.calls == ["info", "usage"] + assert devices[0].cores_utilization == 42 + assert devices[0].power_used == 250 + + +def test_detect_skips_the_usage_query_when_not_asked(): + det = _RecordingDetector(devices=[_device("GPU-0")], usages=[_usage("GPU-0")]) + + devices = det.detect(usage=False) + + assert det.calls == ["info"] + # The information query owns no usage field, so they keep their defaults. + assert devices[0].cores_utilization == 0 + assert devices[0].power_used is None + + +def test_detect_reports_the_inventory_when_the_usage_query_fails(): + # A card exists whether or not its metrics can be read. detect_devices logs + # a raising detector and moves on, so letting this propagate would report no + # hardware at all on a host whose inventory query just succeeded. + class _FailingUsageDetector(_RecordingDetector): + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + self.calls.append("usage") + msg = "the driver lost a device mid-pass" + raise RuntimeError(msg) + + det = _FailingUsageDetector(devices=[_device("GPU-0")]) + + devices = det.detect() + + assert det.calls == ["info", "usage"] + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].cores_utilization == 0 + + +def test_detect_skips_the_usage_query_without_devices(): + for empty in (None, []): + det = _RecordingDetector(devices=empty) + + assert det.detect() == empty + assert det.calls == ["info"] + + +@pytest.mark.parametrize( + "name, body", + [ + # Neither query implemented. + ("_NoQueryDetector", {}), + # Only the information query: a vendor cannot keep a permissive usage + # default that makes detect() look like it measured something. + ("_InfoOnlyDetector", {"detect_info": lambda _self: None}), + # Only the usage query. + ("_UsageOnlyDetector", {"detect_usage": lambda _self, devices=None: devices}), + ], +) +def test_a_detector_skipping_the_split_cannot_be_instantiated(name, body): + detector_type = type( + name, + (Detector,), + {"is_supported": staticmethod(lambda: True), **body}, + ) + + # Both queries are abstract, so a future vendor fails at construction + # rather than silently returning an unmeasured device at run time. + with pytest.raises(TypeError): + detector_type(ManufacturerEnum.UNKNOWN) + + +def test_a_detector_implementing_both_queries_can_be_instantiated(): + assert _RecordingDetector().detect() is None + + +# --------------------------------------------------------------------------- # +# merge_devices_usage. # +# --------------------------------------------------------------------------- # + + +def test_merge_devices_usage_matches_by_uuid(): + devices = [_device("GPU-0"), _device("GPU-1")] + + # Reversed *and* told apart by their payload: the operator's monitor pass + # returns its own list and every consumer joins it by device identity, + # never by index. Identical payloads would leave a positional join, a + # reversed mapping and a broadcast-to-every-card indistinguishable, so the + # values have to differ for this to pin anything. + merge_devices_usage( + devices, + [_usage("GPU-1", cores_utilization=7), _usage("GPU-0", cores_utilization=42)], + ) + + assert [dev.cores_utilization for dev in devices] == [42, 7] + assert [dev.memory_used for dev in devices] == [1024, 1024] + assert [dev.temperature for dev in devices] == [61, 61] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.UNHEALTHY, + DeviceMemoryStatusEnum.UNHEALTHY, + ] + + +def test_merge_devices_usage_keeps_a_health_verdict_the_usage_never_read(): + # No vendor's health helper returns UNKNOWN -- they answer HEALTHY or + # UNHEALTHY -- so an entry carrying UNKNOWN never read health, and writing it + # would erase the information query's verdict. The CLI renders UNKNOWN as + # ERR too, so the erasure is not even visible as one. + devices = [_device("GPU-0", memory_status=DeviceMemoryStatusEnum.UNHEALTHY)] + usage = _usage("GPU-0") + usage.memory_status = DeviceMemoryStatusEnum.UNKNOWN + + merge_devices_usage(devices, [usage]) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + # Every other usage field still merges. + assert devices[0].cores_utilization == 42 + assert devices[0].temperature == 61 + + +def test_merge_devices_usage_downgrades_a_health_verdict_the_usage_did_read(): + devices = [_device("GPU-0", memory_status=DeviceMemoryStatusEnum.UNHEALTHY)] + usage = _usage("GPU-0") + usage.memory_status = DeviceMemoryStatusEnum.HEALTHY + + merge_devices_usage(devices, [usage]) + + # A card that recovered must be allowed to say so. + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_merge_devices_usage_drops_an_ambiguous_uuid(): + # A driver answering the same id for every card -- rocm-smi's unique id + # reporting 0, say -- leaves two cards indistinguishable. Writing either + # card's metrics onto both is worse than reporting neither, so the merge + # leaves them with what the information query read. + devices = [_device("GPU-0"), _device("GPU-0")] + + merge_devices_usage( + devices, + [_usage("GPU-0", cores_utilization=7), _usage("GPU-0", cores_utilization=42)], + ) + + assert [dev.cores_utilization for dev in devices] == [0, 0] + assert [dev.memory_used for dev in devices] == [0, 0] + + +def test_merge_devices_usage_keeps_the_information_fields(): + devices = [_device("GPU-0")] + usage = _usage("GPU-0") + usage.name = "wrong" + usage.memory = 1 + usage.cores = 1 + usage.index = 7 + + merge_devices_usage(devices, [usage]) + + assert devices[0].name == "NVIDIA A100-SXM4-40GB" + assert devices[0].memory == 40960 + assert devices[0].cores == 6912 + assert devices[0].index == 0 + + +def test_merge_devices_usage_merges_the_mig_devices(): + mig_device = { + "index": 2, + "name": "1g.5gb", + "uuid": "MIG-0-0", + "memory": 4864, + "cores_utilization": 0, + "memory_used": 0, + "memory_status": DeviceMemoryStatusEnum.HEALTHY, + "appendix": {"sliced": True, "mig": True}, + } + devices = [_device("GPU-0", appendix={"mig": True, "mig_devices": [mig_device]})] + + merge_devices_usage(devices, [_usage("MIG-0-0"), _usage("GPU-0")]) + + assert mig_device["cores_utilization"] == 42 + assert mig_device["memory_used"] == 1024 + assert mig_device["temperature"] == 61 + assert mig_device["memory_status"] == DeviceMemoryStatusEnum.UNHEALTHY + # The MIG entry stays an appendix entry, and its own fields stay put. + assert mig_device["name"] == "1g.5gb" + assert mig_device["memory"] == 4864 + assert devices[0].cores_utilization == 42 + + +def test_merge_devices_usage_ignores_an_unknown_uuid(): + devices = [_device("GPU-0")] + + merge_devices_usage(devices, [_usage("GPU-9")]) + + assert devices[0].cores_utilization == 0 + assert devices[0].power_used is None + + +def test_merge_devices_usage_tolerates_nothing_to_merge(): + devices = [_device("GPU-0", appendix=None)] + + assert merge_devices_usage(devices, None) is devices + assert merge_devices_usage(devices, []) is devices + assert merge_devices_usage(None, [_usage("GPU-0")]) is None + + +# --------------------------------------------------------------------------- # +# The vendor detectors, as the expand step leaves them. # +# --------------------------------------------------------------------------- # + + +def test_every_detector_implements_the_split(): + for det in _DETECTORS: + # No vendor overrides detect: its payload is detect_info's, plus a + # usage merge that is a no-op until the vendor migrates. That is what + # keeps detect()'s output identical to the pre-split one. + assert type(det).detect is Detector.detect, det.name + assert type(det).detect_info is not Detector.detect_info, det.name + + +def test_every_detector_accepts_no_usage(): + for det in _DETECTORS: + devices = det.detect(usage=False) + assert devices is None or isinstance(devices, list), det.name + + +def test_no_physical_index_switch_survives(): + # Device.index is the enumeration index now, with no switch to make it the + # driver-physical number. Asserted on the whole environment surface rather + # than on the retired name, so a renamed reincarnation fails as well. + assert [name for name in dir(envs) if "PHYSICAL_INDEX" in name] == [] + + +# --------------------------------------------------------------------------- # +# get_pci_device_name: the pci.ids lookup mirroring the operator's # +# GetPCIDeviceNames / GetName. # +# --------------------------------------------------------------------------- # + +# An extract of the real pci.ids, keeping its exact shape: a class section, a +# comment, vendor lines at column 0, device lines behind one tab, subsystem +# lines behind two. +_PCI_IDS = """\ +#\tList of PCI ID's +# +1002 Advanced Micro Devices, Inc. [AMD/ATI] +\t744c Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M] +\t\t1002 0e0d Radeon RX 7900 XTX +\t\t1eae 7900 RX 7900 XTX Phantom Gaming +\t74a1 Aqua Vanjaram [Instinct MI300X] +1d94 Chengdu Haiguang IC Design Co., Ltd. +\t6210 Kunpeng [K100 AI] +\t\t1d94 6210 K100_AI +10de NVIDIA Corporation +\t2330 GH100 [H100 SXM5 80GB] +C 03 Display controller +\t00 VGA compatible controller +""" + + +@pytest.fixture +def pci_ids(tmp_path, monkeypatch): + """ + Point the pci.ids lookup at a fixture database. + """ + + def _write(content: str | None = _PCI_IDS) -> None: + paths: tuple[str, ...] = () + if content is not None: + path = tmp_path / "pci.ids" + path.write_text(content, encoding="utf-8") + paths = (str(path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + paths, + ) + _load_pci_device_names.cache_clear() + + yield _write + + _load_pci_device_names.cache_clear() + + +def test_get_pci_device_name_resolves_a_vendor_device_pair(pci_ids): + pci_ids() + + assert get_pci_device_name("1002", "74a1") == "Aqua Vanjaram [Instinct MI300X]" + assert get_pci_device_name("1d94", "6210") == "Kunpeng [K100 AI]" + assert get_pci_device_name("10de", "2330") == "GH100 [H100 SXM5 80GB]" + + +def test_get_pci_device_name_normalizes_the_ids(pci_ids): + pci_ids() + + # sysfs hands out "0x1002", the SMI libraries hand out an int, and pci.ids + # is lowercase hexadecimal. + for vendor, device in ( + ("0x1002", "0x74A1"), + ("1002", "74A1"), + (0x1002, 0x74A1), + ): + assert get_pci_device_name(vendor, device) == "Aqua Vanjaram [Instinct MI300X]" + + +def test_get_pci_device_name_prefers_the_subsystem_name(pci_ids): + pci_ids() + + assert ( + get_pci_device_name("1002", "744c", "1eae", "7900") + == "RX 7900 XTX Phantom Gaming" + ) + # An unknown subsystem falls back to the device name, as the operator's + # GetName does. + assert ( + get_pci_device_name("1002", "744c", "1eae", "0000") + == "Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M]" + ) + assert ( + get_pci_device_name("1002", "744c") + == "Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M]" + ) + + +def test_get_pci_device_name_returns_nothing_when_unknown(pci_ids): + pci_ids() + + assert get_pci_device_name("1002", "ffff") == "" + assert get_pci_device_name("ffff", "744c") == "" + assert get_pci_device_name("", "") == "" + # The device class table trailing the vendors closes the vendor section: + # its entries must not be attributed to the last vendor parsed. + assert get_pci_device_name("10de", "00") == "" + + +def test_get_pci_device_name_tolerates_a_missing_database(pci_ids): + pci_ids(content=None) + + assert get_pci_device_name("1002", "74a1") == "" diff --git a/tests/gpustack_runtime/detector/test_samples.py b/tests/gpustack_runtime/detector/test_samples.py new file mode 100644 index 0000000..d37f681 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_samples.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from typing import Any + +import pytest + +from gpustack_runtime.detector import Device +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) + +_SAMPLES_PATH = Path(__file__).parent / "samples" + +_DEVICE_FIELDS = {field.name for field in dataclasses.fields(Device)} + +_RETIRED_APPENDIX_KEYS = ( + # Whole-card reporting: no virtual/PF/VF classification. + "vgpu", + # MIG instances are appendix entries of their card, not devices of their + # own, so they no longer carry a device index per instance handle. + "gpu_instance_index", + "compute_instance_index", +) +""" +Appendix keys no detector emits anymore, which a sample must not resurrect. +""" + + +def _samples() -> list[Path]: + """ + The detect output samples. `topology_output_*.json` describes a Topology, + not a Device, and is out of this guard's scope. + """ + return sorted(_SAMPLES_PATH.glob("detect_output_*.json")) + + +def _load(sample: Path) -> list[dict[str, Any]]: + devices = json.loads(sample.read_text()) + assert devices, f"{sample.name} carries no device" + return devices + + +def _mig_devices(device: dict[str, Any]) -> list[dict[str, Any]]: + return (device.get("appendix") or {}).get("mig_devices") or [] + + +def _assert_deserializes(sample: Path, entry: dict[str, Any]) -> None: + unknown = sorted(set(entry) - _DEVICE_FIELDS) + assert not unknown, f"{sample.name} carries fields Device cannot hold: {unknown}" + + device = Device.from_dict(entry) + assert isinstance(device.manufacturer, ManufacturerEnum) + assert isinstance(device.memory_status, DeviceMemoryStatusEnum) + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_deserializes_into_device(sample: Path): + for entry in _load(sample): + _assert_deserializes(sample, entry) + for mig_device in _mig_devices(entry): + _assert_deserializes(sample, mig_device) + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_carries_no_retired_appendix_key(sample: Path): + for entry in _load(sample): + appendices = [entry.get("appendix") or {}] + appendices += [ + mig_device.get("appendix") or {} for mig_device in _mig_devices(entry) + ] + for appendix in appendices: + retired = sorted(set(appendix) & set(_RETIRED_APPENDIX_KEYS)) + assert not retired, f"{sample.name} carries retired keys: {retired}" + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_keeps_mig_instances_inside_the_card(sample: Path): + devices = _load(sample) + indexes = [entry["index"] for entry in devices] + + for entry in devices: + appendix = entry.get("appendix") or {} + assert not appendix.get("sliced"), ( + f"{sample.name} reports a MIG instance as a device of its own" + ) + if appendix.get("mig"): + assert "mig_devices" in appendix, ( + f"{sample.name} enables MIG without reporting its instances" + ) + for mig_device in _mig_devices(entry): + assert mig_device["appendix"]["sliced"] + # index_mig_devices numbers the instances above every card. + assert mig_device["index"] > max(indexes)