diff --git a/deploy/manifests/kubernetes.yaml b/deploy/manifests/kubernetes.yaml index ec9b76d..c00de7f 100644 --- a/deploy/manifests/kubernetes.yaml +++ b/deploy/manifests/kubernetes.yaml @@ -25,6 +25,17 @@ rules: - "pods/exec" verbs: - "*" + # Allow reading the Events of a workload Pod, so an image-pull failure + # reports the registry error behind a bare "ImagePullBackOff" reason. + # Kept as its own rule with only "get"/"list": the rule above grants "*", + # which would also allow creating and deleting Events. + - apiGroups: + - "" + resources: + - "events" + verbs: + - "get" + - "list" # Allow resolving the vendor RuntimeClass (e.g. "ascend", "nvidia") for # accelerated workload Pods, see GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_CLASS. - apiGroups: diff --git a/gpustack_runtime/cmds/detector.py b/gpustack_runtime/cmds/detector.py index 8b109df..2a0349d 100644 --- a/gpustack_runtime/cmds/detector.py +++ b/gpustack_runtime/cmds/detector.py @@ -26,6 +26,7 @@ class DetectDevicesSubCommand(SubCommand): format: str = "table" watch: int = 0 + no_usage: bool = False @staticmethod def register(parser: _SubParsersAction): @@ -49,19 +50,26 @@ def register(parser: _SubParsersAction): help="Continuously watch for GPU in intervals of N seconds", ) + detect_parser.add_argument( + "--no-usage", + action="store_true", + help="Report inventory only, without querying utilization, temperature or power", + ) + detect_parser.set_defaults(func=DetectDevicesSubCommand) def __init__(self, args: Namespace): self.format = args.format self.watch = args.watch + self.no_usage = args.no_usage def run(self): while True: - devs: Devices = detect_devices(fast=False) + devs: Devices = detect_devices(fast=False, usage=not self.no_usage) print("\033[2J\033[H", end="") match self.format.lower(): case "json": - print(format_devices_json(devs)) + print(format_devices_json(devs, usage=not self.no_usage)) case _: # Group devices by manufacturer. group_devs = group_devices_by_manufacturer(devs) @@ -70,7 +78,9 @@ def run(self): else: # Print each group separately. for devs in group_devs.values(): - print(format_devices_table(devs)) + print( + format_devices_table(devs, usage=not self.no_usage), + ) if not self.watch: break time.sleep(self.watch) @@ -134,11 +144,37 @@ def run(self): print(os.linesep.join(legend_lines)) -def format_devices_json(devs: Devices) -> str: - return json.dumps([dev.to_dict() for dev in devs], indent=2) +_USAGE_ONLY_KEYS = ( + "cores_utilization", + "memory_used", + "memory_utilization", + "temperature", + "power_used", +) +""" +The keys only the usage query fills. `memory_status` is deliberately absent: +the information query reports it too, which is why the table keeps Status. +""" + + +def format_devices_json(devs: Devices, usage: bool = True) -> str: + devs_dict = [dev.to_dict() for dev in devs] + + if not usage: + # Without the usage query these fields hold their defaults, and a + # serialized 0 reads as a real idle measurement. The table drops them + # for that reason, and a machine-readable consumer is the more likely of + # the two to act on it -- so absent, meaning unmeasured. + for dev_dict in devs_dict: + mig_devs_dict = (dev_dict.get("appendix") or {}).get("mig_devices") or [] + for d in [dev_dict, *mig_devs_dict]: + for key in _USAGE_ONLY_KEYS: + d.pop(key, None) + + return json.dumps(devs_dict, indent=2) -def format_devices_table(devs: Devices) -> str: +def format_devices_table(devs: Devices, usage: bool = True) -> str: if not devs: return "No GPUs detected." @@ -150,9 +186,14 @@ def format_devices_table(devs: Devices) -> str: row = [ str(dev.index), dev.name if dev.name else "N/A", - f"{dev.memory_used}MiB / {dev.memory}MiB", - f"{dev.cores_utilization}%", - f"{dev.temperature}C" if dev.temperature is not None else "N/A", + # Without the usage query these fields hold their defaults, and a + # rendered 0 would read as a real idle measurement. Total memory + # and the status stay, being reported by the information query. + f"{dev.memory_used}MiB / {dev.memory}MiB" + if usage + else f"N/A / {dev.memory}MiB", + f"{dev.cores_utilization}%" if usage else "N/A", + f"{dev.temperature}C" if usage and dev.temperature is not None else "N/A", dev.compute_capability if dev.compute_capability else "N/A", "OK" if dev.memory_status == DeviceMemoryStatusEnum.HEALTHY else "ERR", ] diff --git a/gpustack_runtime/deployer/__types__.py b/gpustack_runtime/deployer/__types__.py index c385207..6f6af93 100644 --- a/gpustack_runtime/deployer/__types__.py +++ b/gpustack_runtime/deployer/__types__.py @@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from dataclasses_json import dataclass_json @@ -1130,6 +1130,165 @@ def name_rfc1123_guard(self) -> WorkloadName: return self._name_rfc1123_guard +@dataclass_json +@dataclass +class WorkloadStatusExit: + """ + An exit information of a workload's container. + + Attributes: + name (str): + Name representing the exited target, e.g., human-readable container name. + token (WorkloadOperationToken): + Token of the exited target, e.g., container ID. + exit_code (int | None): + Exit code of the container. + reason (str): + Short reason of the termination or the blocking. + message (str): + Detailed message of the termination or the blocking. + started_at (str): + Start time of the container's last execution. + finished_at (str): + Finish time of the container's last execution. + restart_count (int): + Restart times of the container. + + """ + + name: str + """ + Name representing the exited target, e.g., human-readable container name. + """ + token: WorkloadOperationToken + """ + Token of the exited target, e.g. container ID, + which addresses the same container as the logs/exec operations. + """ + exit_code: int | None = None + """ + Exit code of the container, + None if the container has not terminated, e.g. blocked from starting. + """ + reason: str = "" + """ + Short reason of the termination or the blocking, + e.g. "OOMKilled", "Error", "ImagePullBackOff". + """ + message: str = "" + """ + Detailed message of the termination or the blocking. + """ + started_at: str = "" + """ + Start time of the container's last execution, + empty if the container has never started. + """ + finished_at: str = "" + """ + Finish time of the container's last execution, + empty if the container has not finished. + """ + restart_count: int = 0 + """ + Restart times of the container. + """ + + +_CONTAINER_EXITED_STATUSES = ("exited", "dead", "restarting") +""" +Container statuses carrying an exit code. "restarting" is one of them: a +crash-looping container sits there between attempts with the last attempt's +ExitCode already filled in, and that code is the diagnosis being asked for. +""" + +_CONTAINER_UNSET_TIMESTAMP_PREFIX = "0001-01-01" +""" +What Docker fills an unset timestamp with -- Go's zero time -- rather than +omitting the key, which is why a plain `.get(..., "")` never defaults. +""" + + +def parse_container_exit( + c: Any, + component_name_label: str, +) -> WorkloadStatusExit | None: + """ + Build the exit entry of a container that has terminated. + + Docker and Podman report the same State object -- podman-py is Docker-API + compatible and both call sites inspect a fully reloaded container -- so both + deployers share this. Keeping it in one place is the point: the reason + derivation and the timestamp normalization below are exactly the kind of + detail that silently drifts when it lives in two files. + + Args: + c: + A Docker-API-compatible container to inspect. + component_name_label: + The label the deployer names its components with. + + Returns: + A WorkloadStatusExit if the container has terminated, None if it is + still running or pending and so contributes nothing. + + """ + if c.status not in _CONTAINER_EXITED_STATUSES: + return None + + # Every read is .get()-able: a container whose State is missing keys (or + # missing entirely) must degrade gracefully, never raise. + state = c.attrs.get("State", {}) or {} + + exit_code = state.get("ExitCode") + if state.get("OOMKilled"): + reason = "OOMKilled" + elif state.get("Error"): + reason = "Error" + elif exit_code not in (0, None): + # Docker fills State.Error for a container that failed to *start*, not + # for one that ran and exited non-zero -- the commonest crash of all. + # Without this the caller has an exit code and no reason, so it sets no + # state_message, while the Kubernetes deployer reports "Error" for the + # same event. Same field, same answer, either backend. + reason = "Error" + else: + reason = "" + + return WorkloadStatusExit( + name=c.labels.get(component_name_label, "") or c.name, + token=c.attrs.get("Id", "") or c.name, + exit_code=exit_code, + reason=reason, + message=state.get("Error", ""), + started_at=_parse_container_timestamp(state.get("StartedAt")), + finished_at=_parse_container_timestamp(state.get("FinishedAt")), + restart_count=c.attrs.get("RestartCount", 0), + ) + + +def _parse_container_timestamp(value: str | None) -> str: + """ + Normalize a container's State timestamp to what WorkloadStatusExit promises. + + Args: + value: + The raw timestamp as the container's State reports it. + + Returns: + The timestamp, empty if it never happened. + + """ + if not value or value.startswith(_CONTAINER_UNSET_TIMESTAMP_PREFIX): + # WorkloadStatusExit documents these as empty when they never happened, + # and Go's zero time rendered by a UI reads as "January 1, year 1". + return "" + # Docker reports 9-digit nanoseconds where the Kubernetes deployer formats 6 + # (_TIMESTAMP_FORMAT), and strptime's %f accepts at most 6, so a consumer + # parsing this field would succeed on one backend and raise on the other. + return re.sub(r"(\.\d{6})\d+", r"\1", value) + + @dataclass_json @dataclass class WorkloadStatus: @@ -1153,6 +1312,9 @@ class WorkloadStatus: The operation for the executable containers of the workload. loggable (list[WorkloadStatusOperation]): The operation for the loggable containers of the workload. + exits (list[WorkloadStatusExit]): + The exit information for the terminated or start-blocked containers + of the workload. state (WorkloadStatusStateEnum): Current state of the workload. @@ -1192,6 +1354,12 @@ class WorkloadStatus: """ The operation for the loggable containers of the workload. """ + exits: list[WorkloadStatusExit] | None = field(default_factory=list) + """ + The exit information for the terminated or start-blocked containers + of the workload, one entry per container. + Containers in neither state are not listed. + """ state: WorkloadStatusStateEnum = WorkloadStatusStateEnum.UNKNOWN """ The current state of the workload. @@ -1619,6 +1787,42 @@ def map_backend_visible_devices( ) return ret + def map_visible_devices_ordering( + self, + runtime_envs: list[str], + ) -> dict[str, str]: + """ + Return the device ordering environment variables + for the given runtime visible devices env names. + + Only meaningful for a container seeing every device of the host: + it must number the devices as the detector, the driver and the vendor + tooling do, otherwise an index computed from detection addresses + another device inside the container. + For example, CUDA's default ordering, CUDA_DEVICE_ORDER=FASTEST_FIRST, + sorts the visible devices by a performance heuristic, which reshuffles + the ordinals on a heterogeneous host, + while PCI_BUS_ID sorts by PCI bus id, as NVML and the detector enumerate, + see https://github.com/gpustack/gpustack/issues/6041. + No other manufacturer documents such an ordering switch, + so none of them contributes a variable. + + Args: + runtime_envs: + The runtime visible devices environment variable names. + + Returns: + A dictionary mapping device ordering environment variable names + to corresponding values. + + """ + if any( + self.get_manufacturer(runtime_env) == ManufacturerEnum.NVIDIA + for runtime_env in runtime_envs + ): + return {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"} + return {} + def map_visible_devices_affinities( self, runtime_envs: list[str], diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index fa5e88e..40b1ac1 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -112,9 +112,17 @@ def generate( container_device_nodes = [] - cdn_path = f"/dev/davinci{dev.index}" - if dev.appendix.get("vgpu", False): - cdn_path = f"/dev/vdavinci{dev.index}" + # The device node is numbered by the driver's physical id, which + # Device.index no longer carries: it is the detector's enumeration + # index, i.e. the DCMI logic id here. The two are different + # numbers, so a device without a physical id is skipped instead of + # addressed by the index, which would resolve to another NPU's + # node. The detector already drops such a device; this guards the + # devices a caller passes in. + cdn_number = dev.appendix.get("physical_id") + if cdn_number is None: + continue + cdn_path = f"/dev/davinci{cdn_number}" cdn = device_to_cdi_device_node( path=cdn_path, ) diff --git a/gpustack_runtime/deployer/cdi/iluvatar.py b/gpustack_runtime/deployer/cdi/iluvatar.py index e1d0648..5672be0 100644 --- a/gpustack_runtime/deployer/cdi/iluvatar.py +++ b/gpustack_runtime/deployer/cdi/iluvatar.py @@ -81,8 +81,12 @@ def generate( container_device_nodes = [] + # The device node is numbered by the driver's minor number, which + # Device.index no longer carries: it is the detector's enumeration + # index. Fall back to it for a device whose minor number could not + # be read. cdn = device_to_cdi_device_node( - path=f"/dev/iluvatar{dev.index}", + path=f"/dev/iluvatar{dev.appendix.get('minor_number', dev.index)}", ) if not cdn: continue diff --git a/gpustack_runtime/deployer/cdi/thead.py b/gpustack_runtime/deployer/cdi/thead.py index c2fa9b3..56bda5f 100644 --- a/gpustack_runtime/deployer/cdi/thead.py +++ b/gpustack_runtime/deployer/cdi/thead.py @@ -82,11 +82,29 @@ def generate( container_device_nodes = [] + # Named after the card ordinal, i.e. the enumeration index, not + # after the driver's minor number that the detector records in + # appendix["minor_number"]. Unlike /dev/nvidia{N} and + # /dev/iluvatar{N}, which the minor number does name, the operator + # records T-Head's purely to PROVE this node addresses the card it + # describes, by comparing it against the node's character-device + # minor. cdn = device_to_cdi_device_node( path=f"/dev/alixpu_ppu{dev.index}", ) if not cdn: continue + + # So make that comparison rather than assume it: where the detector + # read a minor, the node this ordinal names must carry it. The two + # numbers are independent -- neither is computed from the other, at + # any offset or none -- so a mismatch means this ordinal addresses a + # neighbouring accelerator, which the operator's allocator refuses + # outright rather than hand over. + dev_minor_number = dev.appendix.get("minor_number") + if dev_minor_number is not None and cdn.minor != dev_minor_number: + continue + all_device_nodes.append(cdn) container_device_nodes.append(cdn) diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index 46a4a71..96ed7db 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -45,6 +45,7 @@ WorkloadStatus, WorkloadStatusOperation, WorkloadStatusStateEnum, + parse_container_exit, ) from .__utils__ import ( _MiB, @@ -279,6 +280,12 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = parse_container_exit(c, _LABEL_COMPONENT_NAME) + if exit_ is not None: + self.exits.append(exit_) + if exit_.reason and not self.state_message: + self.state_message = exit_.message or exit_.reason + self.state = self.parse_state(d_containers) @@ -1111,6 +1118,19 @@ def _create_containers( ) create_options["environment"].update(b_vs) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + if r_v == "all" or privileged: + o_vs = self.map_visible_devices_ordering(runtime_envs) + # Take the ordering as default, + # never overwrite the one declared by the container. + create_options["environment"] = { + **o_vs, + **create_options["environment"], + } + # Configure affinity if applicable. create_options.update( self.map_visible_devices_affinities( diff --git a/gpustack_runtime/deployer/kuberentes.py b/gpustack_runtime/deployer/kuberentes.py index 75ce12a..6437a48 100644 --- a/gpustack_runtime/deployer/kuberentes.py +++ b/gpustack_runtime/deployer/kuberentes.py @@ -5,6 +5,7 @@ import logging import os from dataclasses import dataclass, field +from datetime import datetime, timezone from enum import Enum from pathlib import Path from typing import TYPE_CHECKING @@ -31,6 +32,7 @@ WorkloadOperationToken, WorkloadPlan, WorkloadStatus, + WorkloadStatusExit, WorkloadStatusOperation, WorkloadStatusStateEnum, ) @@ -53,6 +55,29 @@ _LABEL_WORKLOAD = f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/workload" _LABEL_COMPONENT = f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/component" +_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +_IMAGE_PULL_BLOCKED_REASONS = frozenset( + { + "ErrImageNeverPull", + "ErrImagePull", + "ImagePullBackOff", + "InvalidImageName", + "RegistryUnavailable", + }, +) +""" +Container waiting reasons meaning the image can never be pulled as requested, +so the workload has failed instead of merely waiting to start. +""" + +_IMAGE_PULL_FAILED_EVENT_REASON = "Failed" +""" +The reason a kubelet stamps on the Event carrying the registry error, e.g. +"Failed to pull image ...: unauthorized". Any other warning the Pod collected, +a stale FailedScheduling among them, explains something else. +""" + class KubernetesWorkloadServiceTypeEnum(str, Enum): """ @@ -196,6 +221,11 @@ def parse_state( """ match k_pod.status.phase: case "Pending": + # A Pod blocked on an image it can never pull is not waiting for + # anything: report it as failed so the caller stops watching a + # workload that will never become ready. + if KubernetesWorkloadStatus.parse_image_pull_block(k_pod): + return WorkloadStatusStateEnum.FAILED return WorkloadStatusStateEnum.PENDING case "Succeeded": return WorkloadStatusStateEnum.INACTIVE @@ -215,10 +245,171 @@ def parse_state( return WorkloadStatusStateEnum.RUNNING + @staticmethod + def parse_image_pull_block( + k_pod: kubernetes.client.V1Pod, + ) -> kubernetes.client.V1ContainerStateWaiting | None: + """ + Find the container state blocking the workload on an image pull. + + Args: + k_pod: + Pod object from Kubernetes API. + + Returns: + The waiting state of the first container blocked on its image pull, + None if no container is. + + """ + for cs in [ + *(k_pod.status.init_container_statuses or []), + *(k_pod.status.container_statuses or []), + ]: + waiting = cs.state.waiting if cs.state else None + if waiting and waiting.reason in _IMAGE_PULL_BLOCKED_REASONS: + return waiting + return None + + @staticmethod + def _parse_exit( + cs: kubernetes.client.V1ContainerStatus | None, + name: str, + ) -> WorkloadStatusExit | None: + """ + Build the exit entry for a container that has terminated or is blocked + from starting. + + Args: + cs: + Status of the container, None if the Pod reports none yet. + name: + Human-readable name of the container. + + Returns: + A WorkloadStatusExit if the container has terminated or is blocked, + None if it is running or has no state at all, and so contributes + nothing. + + """ + if not cs or not cs.state: + return None + + # Prefer the current termination, and fall back to the previous one, so + # a container that has already restarted still reports why it died. + terminated = cs.state.terminated or ( + cs.last_state.terminated if cs.last_state else None + ) + if terminated: + return WorkloadStatusExit( + name=name, + token=cs.name, + exit_code=terminated.exit_code, + reason=terminated.reason or "", + message=terminated.message or "", + started_at=( + terminated.started_at.strftime(_TIMESTAMP_FORMAT) + if terminated.started_at + else "" + ), + finished_at=( + terminated.finished_at.strftime(_TIMESTAMP_FORMAT) + if terminated.finished_at + else "" + ), + restart_count=cs.restart_count or 0, + ) + + if cs.state.waiting: + # A container blocked from starting has never terminated, + # so it carries a reason but no exit code. + return WorkloadStatusExit( + name=name, + token=cs.name, + reason=cs.state.waiting.reason or "", + message=cs.state.waiting.message or "", + restart_count=cs.restart_count or 0, + ) + + return None + + @staticmethod + def _parse_pod_event_message( + k_pod: kubernetes.client.V1Pod, + core_api: kubernetes.client.CoreV1Api, + ) -> str: + """ + Read the Events of the given Pod and return the most relevant message. + + Args: + k_pod: + Pod object from Kubernetes API. + core_api: + Core API client to read the Events with. + + Returns: + The message of the Pod's latest warning Event, + empty if there is none or if the Events cannot be read. + + """ + field_selector = f"involvedObject.name={k_pod.metadata.name}" + if k_pod.metadata.uid: + # A recreated Pod takes the same name, so selecting by name alone + # also selects its predecessor's Events. + field_selector += f",involvedObject.uid={k_pod.metadata.uid}" + + try: + k_events = core_api.list_namespaced_event( + namespace=k_pod.metadata.namespace, + field_selector=field_selector, + ) + except kubernetes.client.exceptions.ApiException: + # Any API failure degrades to no message rather than propagating: + # this read only enriches a diagnosis the state and the reason + # already carry. The expected one is a 403 on a cluster whose + # manifest predates the Events rule, but a 500 or a timeout is no + # reason to fail a status poll either. + debug_log_exception( + logger, + "Failed to list events of pod %s/%s", + k_pod.metadata.namespace, + k_pod.metadata.name, + ) + return "" + + # The warning Events carry the registry error, e.g. the "Failed" Event + # behind a bare "ImagePullBackOff" reason. + k_warnings = [ + k_event + for k_event in k_events.items or [] + if k_event.type == "Warning" and k_event.message + ] + if not k_warnings: + return "" + + # The API guarantees no ordering and an Event name carries a random + # suffix, so list order is not time order: sort it. An Event carrying no + # timestamp at all sorts first, i.e. loses to any that does. + k_warnings.sort( + key=lambda e: (e.last_timestamp or e.event_time or e.first_timestamp) + or datetime.min.replace(tzinfo=timezone.utc), + ) + + # Among the warnings, the kubelet's image-pull failure is the one being + # diagnosed, so it wins over a later warning about something else. + return next( + ( + k_event.message + for k_event in reversed(k_warnings) + if k_event.reason == _IMAGE_PULL_FAILED_EVENT_REASON + ), + k_warnings[-1].message, + ) + def __init__( self, name: WorkloadName, k_pod: kubernetes.client.V1Pod, + core_api: kubernetes.client.CoreV1Api | None = None, **kwargs, ): created_at = k_pod.metadata.creation_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") @@ -244,6 +435,9 @@ def __init__( self._k_pod = k_pod + k_init_css = {cs.name: cs for cs in k_pod.status.init_container_statuses or []} + k_css = {cs.name: cs for cs in k_pod.status.container_statuses or []} + k_pod_annos = k_pod.metadata.annotations or {} for ci, c in enumerate(k_pod.spec.init_containers or []): cn = k_pod_annos.get(f"{_LABEL_COMPONENT}-init-{ci}-name", c.name) @@ -255,6 +449,10 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = self._parse_exit(k_init_css.get(c.name), cn) + if exit_: + self.exits.append(exit_) + for ci, c in enumerate(k_pod.spec.containers): cn = k_pod_annos.get(f"{_LABEL_COMPONENT}-run-{ci}-name", c.name) op = WorkloadStatusOperation( @@ -264,8 +462,38 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = self._parse_exit(k_css.get(c.name), cn) + if exit_: + self.exits.append(exit_) + self.state = self.parse_state(k_pod) - if k_pod.status.message: + + # Only a Pod failed on an image pull reads Events, which is both the + # verdict's own condition and the traffic guard: any other Pod, running + # or pending, costs no extra API call. + image_pull_block = ( + self.parse_image_pull_block(k_pod) + if self.state == WorkloadStatusStateEnum.FAILED + and k_pod.status.phase == "Pending" + else None + ) + if image_pull_block: + # The waiting reason alone ("ImagePullBackOff") omits the registry + # error, which is the part explaining why the pull fails, so append + # the Pod's Event message to it. This diagnosis is more specific + # than the Pod's own status message, hence it wins. + blocked_message = ": ".join( + p for p in [image_pull_block.reason, image_pull_block.message] if p + ) + event_message = ( + self._parse_pod_event_message(k_pod, core_api) if core_api else "" + ) + self.state_message = ( + f"{blocked_message}; {event_message}" + if event_message + else blocked_message + ) + elif k_pod.status.message: # Surface the Pod's status message, e.g. a device-plugin # admission rejection ("UnexpectedAdmissionError: Allocate # failed ...") on Failed Pods. @@ -1318,6 +1546,29 @@ def _create_pod( ], ) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + # This includes requesting all devices under KDP: + # the device plugin allocates every device of the node, + # hence the container still enumerates all of them. + # Never overwrite the ordering declared by the container. + if r_v == "all" or privileged: + declared_envs = {e.name for e in container.env} + container.env.extend( + [ + kubernetes.client.V1EnvVar( + name=o_k, + value=o_v, + ) + for o_k, o_v in self.map_visible_devices_ordering( + runtime_envs, + ).items() + if o_k not in declared_envs + ], + ) + container.resources = kubernetes.client.V1ResourceRequirements( limits=(resources if resources else None), requests=(resources if resources else None), @@ -1860,6 +2111,7 @@ def _get( return KubernetesWorkloadStatus( name=name, k_pod=k_pod, + core_api=core_api, ) @_supported @@ -2041,6 +2293,7 @@ def _list( KubernetesWorkloadStatus( name=k_pod.metadata.labels[_LABEL_WORKLOAD], k_pod=k_pod, + core_api=core_api, ) for k_pod in k_pods.items or [] if ( diff --git a/gpustack_runtime/deployer/podman.py b/gpustack_runtime/deployer/podman.py index 7302a3c..eefe869 100644 --- a/gpustack_runtime/deployer/podman.py +++ b/gpustack_runtime/deployer/podman.py @@ -48,6 +48,7 @@ WorkloadStatus, WorkloadStatusOperation, WorkloadStatusStateEnum, + parse_container_exit, ) from .__utils__ import ( _MiB, @@ -282,6 +283,12 @@ def __init__( self.executable.append(op) self.loggable.append(op) + exit_ = parse_container_exit(c, _LABEL_COMPONENT_NAME) + if exit_ is not None: + self.exits.append(exit_) + if exit_.reason and not self.state_message: + self.state_message = exit_.message or exit_.reason + self.state = self.parse_state(d_containers) @@ -1090,6 +1097,19 @@ def _create_containers( ) create_options["environment"].update(b_vs) + # If requesting all devices or privileged, + # the container sees every device of the host, + # so pin the device ordering to keep its numbering + # aligned with the detection. + if r_v == "all" or privileged: + o_vs = self.map_visible_devices_ordering(runtime_envs) + # Take the ordering as default, + # never overwrite the one declared by the container. + create_options["environment"] = { + **o_vs, + **create_options["environment"], + } + # Configure affinity if applicable. create_options.update( self.map_visible_devices_affinities( diff --git a/gpustack_runtime/detector/__init__.py b/gpustack_runtime/detector/__init__.py index 7a7d6db..b21ae64 100644 --- a/gpustack_runtime/detector/__init__.py +++ b/gpustack_runtime/detector/__init__.py @@ -141,6 +141,7 @@ def detect_backend( def detect_devices( fast: bool = True, manufacturer: ManufacturerEnum = None, + usage: bool = True, ) -> Devices: """ Detect all available devices. @@ -152,6 +153,8 @@ def detect_devices( manufacturer: Manufacturer to filter the detection, implies `fast=True`. If None, detect all available manufacturers. + usage: + If True, fetch the devices' usage as well. Returns: A list of detected devices. @@ -165,7 +168,7 @@ def detect_devices( det = _DETECTORS_MAP.get(manufacturer) if det and det.is_supported(): try: - return det.detect() + return det.detect(usage=usage) except Exception: detect_target = envs.GPUSTACK_RUNTIME_DETECT.lower() if detect_target == det.name: @@ -180,7 +183,7 @@ def detect_devices( continue try: - if devs := det.detect(): + if devs := det.detect(usage=usage): devices.extend(devs) if fast and devices: return devices @@ -219,7 +222,9 @@ def get_devices_topologies( """ group = False if not devices: - devices = detect_devices(fast=fast, manufacturer=manufacturer) + # Topology keys off identity and the appendix only, so the usage query + # is not worth its cost here. + devices = detect_devices(fast=fast, manufacturer=manufacturer, usage=False) if not devices: return [] group = not fast diff --git a/gpustack_runtime/detector/__types__.py b/gpustack_runtime/detector/__types__.py index f28e0fc..32b854f 100644 --- a/gpustack_runtime/detector/__types__.py +++ b/gpustack_runtime/detector/__types__.py @@ -1,5 +1,6 @@ from __future__ import annotations as __future_annotations__ +import logging from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum @@ -8,6 +9,10 @@ from dataclasses_json import dataclass_json +from ..logging import debug_log_exception, debug_log_warning + +logger = logging.getLogger(__name__) + class ManufacturerEnum(str, Enum): """ @@ -157,11 +162,9 @@ class Device: """ index: int = 0 """ - Index of the device. - If GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY is set to 1, - this will be the physical index of the device. - Otherwise, it will be the logical index of the device. - Physical index is adapted to non-virtualized devices. + Index of the device, as the detector enumerates it. + Driver-physical numbering, which a device node path or a vendor tool needs, + lives in `appendix` instead, e.g. `minor_number` or `card_id`/`physical_id`. """ name: str = "" """ @@ -471,11 +474,10 @@ def index_mig_devices( A MIG device carries no driver-side inventory index, so its index is synthetic: every card owns a block of `slots` indexes, and the blocks - start above the largest index the physical cards report. The reported - index may be the card's minor number - (`GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY`), which is not bound by - the card count, hence the offset is measured from the reported indexes - instead of the count. Sizing a block by the slots a card can host, rather + start above the largest index the physical cards report. A detector's + reported index is not necessarily zero-based and contiguous — Ascend + reports the DCMI logic id — hence the offset is measured from the reported + indexes instead of the card count. Sizing a block by the slots a card can host, rather than by the MIG devices it currently has, keeps a card's numbering independent of its neighbours: partitioning one card never renumbers another's MIG devices. @@ -497,6 +499,113 @@ def index_mig_devices( mig["index"] += base + dev_idx * slots +_DEVICE_USAGE_FIELDS = ( + "cores_utilization", + "memory_used", + "memory_utilization", + "memory_status", + "temperature", + "power_used", +) +""" +The fields the usage query owns, i.e. everything a detector must re-read to +refresh a device it already knows. `memory_status` belongs to both queries, +mirroring the operator, which reports health from `DetectAccelerator` and +`MonitorAccelerator` alike. +""" + + +def merge_devices_usage( + devices: Devices | None, + usages: Devices | None, +) -> Devices | None: + """ + Merge the given usage into the given devices in place, matched by UUID. + + The usage query returns devices of its own, keyed by UUID, which this + joins into the devices to refresh: the operator does the same, its + `MonitorAccelerator` returning a separate metrics list that every consumer + joins by device identity and never by index, as an index is not stable + across a re-detection. + + MIG devices are refreshed as well: they live in the card's + `appendix["mig_devices"]` and carry their own UUID, so a usage entry + matching one is merged into that entry. + + Args: + devices: + The devices to refresh. + usages: + The devices carrying the usage to merge. + + Returns: + The given devices, refreshed. + + """ + if not devices or not usages: + return devices + + # The join is only as good as the identities behind it, and a driver + # answering the same id for every card is how that breaks: taking either + # entry would write one card's utilization, memory and temperature onto + # another. An ambiguous id is therefore dropped rather than guessed at, + # leaving those cards with what the information query read. + usages_map: dict[str, Device] = {} + ambiguous_uuids: set[str] = set() + for usage in usages: + if not usage.uuid: + continue + if usage.uuid in usages_map: + ambiguous_uuids.add(usage.uuid) + continue + usages_map[usage.uuid] = usage + for uuid in ambiguous_uuids: + debug_log_warning( + logger, + "Skipping usage of uuid %s, reported by more than one device", + uuid, + ) + del usages_map[uuid] + + for dev in devices: + if usage := usages_map.get(dev.uuid): + for field in _usage_fields_to_merge(usage): + setattr(dev, field, getattr(usage, field)) + + # Keyed by the instance's own UUID, so what lands in a MIG entry is the + # usage query's reading for that instance -- the entry is written, never + # read from. + for mig_dev in (dev.appendix or {}).get("mig_devices") or []: + if mig_usage := usages_map.get(mig_dev.get("uuid")): + for field in _usage_fields_to_merge(mig_usage): + mig_dev[field] = getattr(mig_usage, field) + + return devices + + +def _usage_fields_to_merge(usage: Device) -> tuple[str, ...]: + """ + Select the fields of the given usage entry that are worth writing over a device. + + Args: + usage: + The device carrying the usage to merge. + + Returns: + The names of the fields to copy. + + """ + if usage.memory_status != DeviceMemoryStatusEnum.UNKNOWN: + return _DEVICE_USAGE_FIELDS + + # No vendor's health helper returns UNKNOWN: they answer HEALTHY or + # UNHEALTHY, and HEALTHY when the check is switched off. So an entry carrying + # UNKNOWN never read health, and writing it would erase the verdict the + # information query found -- which the CLI renders as ERR, exactly as it + # renders UNHEALTHY. + return tuple(field for field in _DEVICE_USAGE_FIELDS if field != "memory_status") + + class Detector(ABC): """ Base class for all detectors. @@ -537,15 +646,62 @@ def name(self) -> str: return str(self.manufacturer) @abstractmethod - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: + """ + Detect devices' inventory, without usage metrics. + + Returns: + A list of detected Device objects, or None if not supported. + + """ + raise NotImplementedError + + @abstractmethod + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of the given devices, merged into them in place. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + """ + raise NotImplementedError + + def detect(self, usage: bool = True) -> Devices | None: """ Detect devices and return a list of Device objects. + Args: + usage: + Whether to fetch the devices' usage as well. + Returns: A list of detected Device objects, or None if detection fails. """ - raise NotImplementedError + devices = self.detect_info() + if usage and devices: + try: + self.detect_usage(devices) + except Exception: + # The inventory is already in hand, and a card exists whether or + # not its metrics could be read. Letting this propagate costs + # every device of the vendor -- detect_devices logs and moves on + # -- so a host with eight healthy cards reports none because one + # metric query failed. The usage fields keep what the + # information query read. + debug_log_exception( + logger, + "Failed to fetch %s devices usage, reporting information only", + self.manufacturer, + ) + return devices def get_topology(self, devices: Devices | None = None) -> Topology | None: """ diff --git a/gpustack_runtime/detector/__utils__.py b/gpustack_runtime/detector/__utils__.py index e50b742..e3a2e6d 100644 --- a/gpustack_runtime/detector/__utils__.py +++ b/gpustack_runtime/detector/__utils__.py @@ -1013,24 +1013,149 @@ def bitmask_to_str(bitmask_list: list) -> str: return list_to_str_range(sorted(bits_lists)) -def get_physical_function_by_bdf(bdf: str) -> str: +_PCI_IDS_PATHS: tuple[str, ...] = ( + "/usr/share/hwdata/pci.ids", + "/usr/share/pci.ids", + "/usr/share/misc/pci.ids", +) +""" +Candidate locations of the PCI ID database, +in the order the operator's GetPCIDeviceNames probes them. +""" + + +def _normalize_pci_id(value: int | str | None) -> str: """ - Get the physical function BDF for a given PCI device BDF address. + Normalize a PCI ID to the lowercase hexadecimal form the PCI ID database uses. Args: - bdf: - The PCI device BDF address (e.g., "0000:00:1f.0"). + value: + The PCI ID, e.g. 0x1002, "0x1002" or "1002". + sysfs reports it prefixed, the vendor SMI libraries report an integer. Returns: - The physical function BDF if found, otherwise returns the original BDF. + The normalized PCI ID, or an empty string if there is none. """ - if bdf: - with contextlib.suppress(Exception): - dev_path = Path(f"/sys/bus/pci/devices/{bdf}") - if dev_path.exists(): - physfn_path = dev_path / "physfn" - if physfn_path.exists(): - physfn_realpath = physfn_path.resolve() - return physfn_realpath.name - return bdf + if value is None: + return "" + if isinstance(value, int): + return f"{value:04x}" + return value.strip().lower().removeprefix("0x") + + +@lru_cache +def _load_pci_device_names( + vendor: str, +) -> dict[str, tuple[str, dict[tuple[str, str], str]]]: + """ + Parse the PCI ID database for one vendor. + + Mirrors the parsing of the operator's GetPCIDeviceNames: a vendor line + starts at column 0, a device line behind one tab, a subsystem line behind + two. The database also holds a device class table ("C 03 Display + controller") and comments, which are read as vendor lines and never match + a real vendor, so they gate off the same way an unrequested vendor does. + + Args: + vendor: + The normalized PCI vendor ID to parse for. + + Returns: + A mapping of PCI device ID to the device's name and its subsystem + names, keyed by (subsystem vendor ID, subsystem device ID). + Empty if the database or the vendor is not found. + + """ + names: dict[str, tuple[str, dict[tuple[str, str], str]]] = {} + if not vendor: + return names + + path = next((p for p in map(Path, _PCI_IDS_PATHS) if p.exists()), None) + if not path: + return names + + in_vendor = False + device = "" + with ( + contextlib.suppress(OSError), + path.open("r", encoding="utf-8", errors="ignore") as f, + ): + for raw_line in f: + line = raw_line.rstrip("\n") + if not line: + continue + + fields = line.split() + + # A vendor line, which switches the section being parsed. A line + # holding a single field leaves the section as it is, as the + # operator does. + if not line.startswith("\t"): + if len(fields) > 1: + in_vendor = fields[0].lower() == vendor + device = "" + continue + + if not in_vendor: + continue + + # A device line. + if not line.startswith("\t\t"): + if len(fields) > 1: + device = fields[0].lower() + names[device] = (" ".join(fields[1:]), {}) + continue + + # A subsystem line, belonging to the device line above it. + if device and len(fields) > 2: + names[device][1][(fields[0].lower(), fields[1].lower())] = " ".join( + fields[2:], + ) + + return names + + +def get_pci_device_name( + vendor: int | str, + device: int | str, + subvendor: int | str = "", + subdevice: int | str = "", +) -> str: + """ + Get the name of a PCI device from the local PCI ID database. + + Mirrors the operator's GetPCIDeviceNames/GetName, which prefers the + subsystem name over the device name: a board vendor's name for the card is + more precise than the chip's. + + Args: + vendor: + The PCI vendor ID, e.g. 0x1002, "0x1002" or "1002". + device: + The PCI device ID. + subvendor: + The PCI subsystem vendor ID, if any. + subdevice: + The PCI subsystem device ID, if any. + + Returns: + The name of the device, + or an empty string if the database or the device is not found. + + """ + vendor = _normalize_pci_id(vendor) + device = _normalize_pci_id(device) + if not vendor or not device: + return "" + + entry = _load_pci_device_names(vendor).get(device) + if not entry: + return "" + + name, subnames = entry + subvendor = _normalize_pci_id(subvendor) + subdevice = _normalize_pci_id(subdevice) + if subvendor and subdevice: + return subnames.get((subvendor, subdevice), name) + return name diff --git a/gpustack_runtime/detector/amd.py b/gpustack_runtime/detector/amd.py index f905e45..d8e1f8b 100644 --- a/gpustack_runtime/detector/amd.py +++ b/gpustack_runtime/detector/amd.py @@ -15,6 +15,7 @@ Devices, ManufacturerEnum, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -22,14 +23,26 @@ compare_pci_devices, get_brief_version, get_numa_node_by_bdf, + get_pci_device_name, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) logger = logging.getLogger(__name__) +_PCI_DEVICES_PATH = Path("/sys/bus/pci/devices") +""" +Location where sysfs exposes the PCI devices, +which a device's PCI IDs are read from. +""" + +_PCI_ID_FILES = ("vendor", "device", "subsystem_vendor", "subsystem_device") +""" +The sysfs files holding a PCI device's IDs, +in the order the PCI ID database is queried with. +""" + class AMDDetector(Detector): """ @@ -76,9 +89,10 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.AMD) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect AMD GPUs using pyamdsmi, pyamdgpu and pyrocmsmi. + Detect AMD GPUs' inventory using pyamdsmi, pyamdgpu and pyrocmsmi, + without usage metrics. Returns: A list of detected AMD GPU devices, @@ -113,11 +127,7 @@ def detect(self) -> Devices | None: dev_index = dev_idx dev_gpu_asic_info = pyamdsmi.amdsmi_get_gpu_asic_info(dev) - if dev_gpu_asic_info.get("asic_serial") != "N/A": - asic_serial = dev_gpu_asic_info.get("asic_serial") - dev_uuid = f"GPU-{(asic_serial[2:]).lower()}" - else: - dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + dev_uuid = _get_device_uuid(dev_gpu_asic_info, dev_idx) dev_bdf = pyamdsmi.amdsmi_get_gpu_device_bdf(dev) dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf) @@ -129,7 +139,23 @@ def detect(self) -> Devices | None: dev_gpu_driver_info = pyamdsmi.amdsmi_get_gpu_driver_info(dev) dev_driver_ver = dev_gpu_driver_info.get("driver_version") - dev_name = dev_hsa_agent.name + # The operator resolves the name from the local PCI ID database + # first: pci.ids knows the board -- the subsystem vendor's name + # for the card -- where the driver only knows the chip. The + # driver-reported names stay as fallbacks. + dev_name = _get_pci_device_name_by_bdf(dev_bdf) + if not dev_name: + dev_name = dev_hsa_agent.name + if not dev_name and dev_card_id is not None: + # The operator asks libdrm for the board's marketing name + # between the HSA and the ASIC name, so this is that step. + # Reached only when the two above found nothing, which is + # why the device is opened here rather than up front. + with ( + contextlib.suppress(pyamdgpu.AMDGPUError), + pyamdgpu.amdgpu_device(dev_card_id) as dev_gpudev, + ): + dev_name = pyamdgpu.amdgpu_get_marketing_name(dev_gpudev) if not dev_name: dev_name = dev_gpu_asic_info.get("market_name") @@ -158,31 +184,11 @@ def detect(self) -> Devices | None: if not dev_asic_family_id: dev_asic_family_id = dev_gpudev_info.family_id - dev_cores_util = None - dev_temp = None - try: - dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev) - dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0) - dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0) - except pyamdsmi.AmdSmiException: - with contextlib.suppress(pyrocmsmi.ROCMSMIError): - dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) - dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - dev_mem = 0 - dev_mem_used = 0 dev_mem_status = DeviceMemoryStatusEnum.HEALTHY try: dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) dev_mem = dev_gpu_vram_usage.get("vram_total") - dev_mem_used = dev_gpu_vram_usage.get("vram_used") if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( dev, @@ -194,9 +200,6 @@ def detect(self) -> Devices | None: dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), - ) if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyrocmsmi.ROCMSMIError): dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( @@ -205,24 +208,17 @@ def detect(self) -> Devices | None: if dev_ecc_count.uncorrectable_err > 0: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # The power limit is inventory, so it stays here, while the used + # power the same call carries belongs to the usage query. dev_power = None - dev_power_used = None try: dev_power_info = pyamdsmi.amdsmi_get_power_info(dev) dev_power = ( dev_power_info.get("power_limit", 0) // 1000000 ) # uW to W - dev_power_used = ( - dev_power_info.get("current_socket_power") - if dev_power_info.get("current_socket_power", "N/A") != "N/A" - else dev_power_info.get("average_socket_power", 0) - ) except pyamdsmi.AmdSmiException: with contextlib.suppress(pyrocmsmi.ROCMSMIError): dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx) - dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) - - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: @@ -231,7 +227,6 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _get_arch_family(dev_asic_family_id), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -255,14 +250,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -275,6 +265,126 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch AMD GPUs' usage using pyamdsmi and pyrocmsmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyamdsmi.amdsmi_init() + try: + pyrocmsmi.rsmi_init() + except Exception: + debug_log_exception(logger, "Failed to initialize ROCm SMI") + + devs = pyamdsmi.amdsmi_get_processor_handles() + for dev_idx, dev in enumerate(devs): + dev_gpu_asic_info = pyamdsmi.amdsmi_get_gpu_asic_info(dev) + dev_uuid = _get_device_uuid(dev_gpu_asic_info, dev_idx) + + dev_cores_util = None + dev_temp = None + try: + dev_gpu_metrics_info = pyamdsmi.amdsmi_get_gpu_metrics_info(dev) + dev_cores_util = dev_gpu_metrics_info.get("average_gfx_activity", 0) + dev_temp = dev_gpu_metrics_info.get("temperature_hotspot", 0) + except pyamdsmi.AmdSmiException: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) + dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem = 0 + dev_mem_used = 0 + # Health is reported by both queries, as the operator reports it + # from DetectAccelerator and MonitorAccelerator alike. + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + try: + dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) + dev_mem = dev_gpu_vram_usage.get("vram_total") + dev_mem_used = dev_gpu_vram_usage.get("vram_used") + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( + dev, + pyamdsmi.AmdSmiGpuBlock.UMC, + ) + if dev_ecc_count.get("uncorrectable_count", 0) > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + except pyamdsmi.AmdSmiException: + dev_mem = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), + ) + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( + dev_idx, + ) + if dev_ecc_count.uncorrectable_err > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_power_used = None + try: + dev_power_info = pyamdsmi.amdsmi_get_power_info(dev) + dev_power_used = ( + dev_power_info.get("current_socket_power") + if dev_power_info.get("current_socket_power", "N/A") != "N/A" + else dev_power_info.get("average_socket_power", 0) + ) + except pyamdsmi.AmdSmiException: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyamdsmi.AmdSmiException: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between AMD GPUs. @@ -289,7 +399,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -400,6 +510,56 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum: return ret +def _get_pci_device_name_by_bdf(dev_bdf: str) -> str: + """ + Get the name of a device from the local PCI ID database. + + Mirrors the operator, which prefers this name over every driver-reported + one: pci.ids knows the board -- the subsystem vendor's name for the card -- + where the driver only knows the chip. + + Args: + dev_bdf: + The device bdf. + + Returns: + The name of the device, + or an empty string if the database or the device is unknown. + + """ + if not dev_bdf: + return "" + + dev_pci_ids = [] + for id_file in _PCI_ID_FILES: + dev_pci_id = "" + with contextlib.suppress(OSError): + dev_pci_id = (_PCI_DEVICES_PATH / dev_bdf / id_file).read_text().strip() + dev_pci_ids.append(dev_pci_id) + + return get_pci_device_name(*dev_pci_ids) + + +def _get_device_uuid(dev_gpu_asic_info: dict, dev_idx: int) -> str: + """ + Get the UUID of a device. + + Args: + dev_gpu_asic_info: + The ASIC information of the device. + dev_idx: + The index of the device. + + Returns: + The UUID of the device. + + """ + if dev_gpu_asic_info.get("asic_serial") != "N/A": + asic_serial = dev_gpu_asic_info.get("asic_serial") + return f"GPU-{(asic_serial[2:]).lower()}" + return f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + + def _get_arch_family(dev_family_id: int | None) -> str: """ Get the architecture family name from the device family ID. diff --git a/gpustack_runtime/detector/ascend.py b/gpustack_runtime/detector/ascend.py index a88fe31..0f6208f 100644 --- a/gpustack_runtime/detector/ascend.py +++ b/gpustack_runtime/detector/ascend.py @@ -18,6 +18,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -92,9 +93,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.ASCEND) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Ascend NPUs using pydcmi. + Detect Ascend NPUs' inventory using pydcmi, without usage metrics. Returns: A list of detected Ascend NPU devices, @@ -121,82 +122,54 @@ def detect(self) -> Devices | None: for dev_card_id in card_list: device_num_in_card = pydcmi.dcmi_get_device_num_in_card(dev_card_id) for dev_device_id in range(device_num_in_card): - dev_is_vgpu = False - dev_virt_info = _get_device_virtual_info( + if not _is_npu_device(dev_card_id, dev_device_id): + continue + + dev_chip_info = _get_device_chip_info( dev_card_id, dev_device_id, ) - if ( - dev_virt_info - and hasattr(dev_virt_info, "query_info") - and hasattr(dev_virt_info.query_info, "computing") - ): - dev_is_vgpu = True - dev_cores_aicore = dev_virt_info.query_info.computing.aic - dev_name = dev_virt_info.query_info.name - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - if hasattr(dev_virt_info.query_info.computing, "memory_size"): - dev_mem = dev_virt_info.query_info.computing.memory_size - dev_index = dev_virt_info.vdev_id - else: - dev_chip_info = pydcmi.dcmi_get_device_chip_info_v2( - dev_card_id, - dev_device_id, - ) - dev_cores_aicore = dev_chip_info.aicore_cnt - dev_name = dev_chip_info.chip_name - dev_mem, dev_mem_used = _get_device_memory_info( - dev_card_id, - dev_device_id, - ) - dev_mem_status = _get_device_memory_status( - dev_card_id, - dev_device_id, - ) - dev_index = pydcmi.dcmi_get_device_logic_id( - dev_card_id, - dev_device_id, - ) - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - dev_index = pydcmi.dcmi_get_device_phyid_from_logicid( - dev_index, - ) - dev_uuid = pydcmi.dcmi_get_device_die_v2( + dev_cores_aicore = dev_chip_info.aicore_cnt + dev_name = dev_chip_info.chip_name + dev_mem, _ = _get_device_memory_info( dev_card_id, dev_device_id, - pydcmi.DCMI_DIE_TYPE_VDIE, ) - - dev_util_aicore = pydcmi.dcmi_get_device_utilization_rate( + dev_mem_status = _get_device_memory_status( dev_card_id, dev_device_id, - pydcmi.DCMI_INPUT_TYPE_AICORE, ) - if dev_util_aicore is None: + dev_index = pydcmi.dcmi_get_device_logic_id( + dev_card_id, + dev_device_id, + ) + # Device.index is the logic id the driver enumerates + # the NPU at, while the physical id is what a device + # node path is made of, so the latter goes to the + # appendix beside the card and device ids. Mirrors the + # operator, which keeps a sequential Index next to + # PhysicalIndexes. + # + # A device whose physical id cannot be read cannot be + # addressed at all: /dev/davinciN is numbered by it, and + # the logic id is a different number, so standing in for + # it would hand a container another NPU's node. The + # operator skips such a device for the same reason. + try: + dev_physical_id = pydcmi.dcmi_get_device_phyid_from_logicid( + dev_index, + ) + except pydcmi.DCMIError: debug_log_warning( logger, - "Failed to get device %d cores utilization, setting to 0", + "Failed to fetch physical id of device %d, skipping it", dev_index, ) - dev_util_aicore = 0 - - dev_temp = pydcmi.dcmi_get_device_temperature( - dev_card_id, - dev_device_id, - ) + continue - dev_power_used = None - with contextlib.suppress(pydcmi.DCMIError): - dev_power_used = pydcmi.dcmi_get_device_power_info( - dev_card_id, - dev_device_id, - ) - if dev_power_used: - dev_power_used = dev_power_used / 10 # 0.1W to W + dev_uuid = _get_device_die(dev_card_id, dev_device_id) - dev_bdf = pydcmi.dcmi_get_device_bdf( + dev_bdf = _get_device_bdf( dev_card_id, dev_device_id, ) @@ -214,11 +187,11 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _guess_soc_name_from_dev_name(dev_name), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, "card_id": dev_card_id, "device_id": dev_device_id, "device_id_max": device_num_in_card - 1, + "physical_id": dev_physical_id, } if dev_numa: dev_appendix["numa"] = dev_numa @@ -246,13 +219,8 @@ def detect(self) -> Devices | None: runtime_version=sys_runtime_ver, runtime_version_original=sys_runtime_ver_original, cores=dev_cores_aicore, - cores_utilization=dev_util_aicore, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -265,6 +233,110 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Ascend NPUs' usage using pydcmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pydcmi.dcmi_init() + + _, card_list = pydcmi.dcmi_get_card_list() + for dev_card_id in card_list: + device_num_in_card = pydcmi.dcmi_get_device_num_in_card(dev_card_id) + for dev_device_id in range(device_num_in_card): + # The operator filters by device type in MonitorAccelerator + # as well, not only when detecting. + if not _is_npu_device(dev_card_id, dev_device_id): + continue + + dev_uuid = _get_device_die(dev_card_id, dev_device_id) + + # The operator's MonitorAccelerator re-reads the memory + # rather than trusting the detection pass. + dev_mem, dev_mem_used = _get_device_memory_info( + dev_card_id, + dev_device_id, + ) + dev_mem_status = _get_device_memory_status( + dev_card_id, + dev_device_id, + ) + + dev_util_aicore = None + with contextlib.suppress(pydcmi.DCMIError): + dev_util_aicore = pydcmi.dcmi_get_device_utilization_rate( + dev_card_id, + dev_device_id, + pydcmi.DCMI_INPUT_TYPE_AICORE, + ) + if dev_util_aicore is None: + debug_log_warning( + logger, + "Failed to get device %d/%d cores utilization, " + "setting to 0", + dev_card_id, + dev_device_id, + ) + dev_util_aicore = 0 + + dev_temp = None + with contextlib.suppress(pydcmi.DCMIError): + dev_temp = pydcmi.dcmi_get_device_temperature( + dev_card_id, + dev_device_id, + ) + + dev_power_used = None + with contextlib.suppress(pydcmi.DCMIError): + dev_power_used = pydcmi.dcmi_get_device_power_info( + dev_card_id, + dev_device_id, + ) + if dev_power_used: + dev_power_used = dev_power_used / 10 # 0.1W to W + + usages.append( + Device( + uuid=dev_uuid.upper(), + cores_utilization=dev_util_aicore, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pydcmi.DCMIError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between Ascend NPUs. @@ -279,7 +351,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -340,6 +412,118 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _is_npu_device(dev_card_id, dev_device_id) -> bool: + """ + Report whether the given device of the card is an NPU. + + A card also carries non-NPU units, like its MCU, which are not + accelerators. Mirrors the operator, which skips a device only when the + type call *succeeds* and reports something other than an NPU: a device + whose type cannot be read is kept. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + True if the device is an NPU, or its type is unreadable. + + """ + dev_type = None + with contextlib.suppress(pydcmi.DCMIError): + dev_type = pydcmi.dcmi_get_device_type(dev_card_id, dev_device_id) + + if dev_type is not None and dev_type != pydcmi.DCMI_UNIT_TYPE_NPU: + slogger.debug( + "Skipping non-NPU device %d of card %d, type %d", + dev_device_id, + dev_card_id, + dev_type, + ) + return False + + return True + + +def _get_device_die(dev_card_id, dev_device_id) -> str: + """ + Get the device's SoC die, which identifies it. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The die as a string. + + """ + try: + return pydcmi.dcmi_get_device_die_v2( + dev_card_id, + dev_device_id, + pydcmi.DCMI_DIE_TYPE_VDIE, + ) + except pydcmi.DCMIError: + # An older driver exposes the V1 call only, which takes no die type + # and reports the SoC die directly. Mirrors the operator's + # VDieHandler, which tries V2 then V1. + return pydcmi.dcmi_get_device_die(dev_card_id, dev_device_id) + + +def _get_device_bdf(dev_card_id, dev_device_id) -> str: + """ + Get the device's PCI bus address. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The BDF as a string. + + """ + try: + return pydcmi.dcmi_get_device_bdf(dev_card_id, dev_device_id) + except pydcmi.DCMIError: + # An older driver exposes the V1 PCIe call only, whose struct carries + # no PCI domain, so the domain reads as 0 -- as the operator's + # PcieInfoHandler.V1 leaves it when widening to the V2 struct. + dev_pcie_info = pydcmi.dcmi_get_device_pcie_info(dev_card_id, dev_device_id) + return ( + f"0000:{dev_pcie_info.bdf_busid:02x}:" + f"{dev_pcie_info.bdf_deviceid:02x}.{dev_pcie_info.bdf_funcid:x}" + ) + + +def _get_device_chip_info(dev_card_id, dev_device_id): + """ + Get the device's chip information. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + The chip information, carrying at least chip_name and aicore_cnt. + + """ + try: + return pydcmi.dcmi_get_device_chip_info_v2(dev_card_id, dev_device_id) + except pydcmi.DCMIError: + # The binding's V2 wrapper already falls back when the symbol is + # missing; the operator's ChipInfoHandler falls back on any failure, + # which an older driver rejecting the V2 struct produces. + return pydcmi.dcmi_get_device_chip_info(dev_card_id, dev_device_id) + + def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]: """ Get device memory information. @@ -357,35 +541,57 @@ def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]: try: dev_hbm_info = pydcmi.dcmi_get_device_hbm_info(dev_card_id, dev_device_id) if dev_hbm_info.memory_size > 0: - dev_mem = dev_hbm_info.memory_size - dev_mem_used = dev_hbm_info.memory_usage - else: - dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( - dev_card_id, - dev_device_id, - ) - dev_mem = dev_memory_info.memory_size - dev_mem_used = ( - dev_memory_info.memory_size - dev_memory_info.memory_available - ) + return dev_hbm_info.memory_size, dev_hbm_info.memory_usage except pydcmi.DCMIError as e: - if e.value in [ + if e.value not in [ pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND, pydcmi.DCMI_ERROR_NOT_SUPPORT, pydcmi.DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER, ]: - dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( - dev_card_id, - dev_device_id, - ) - dev_mem = dev_memory_info.memory_size - dev_mem_used = ( - dev_memory_info.memory_size - dev_memory_info.memory_available - ) - else: raise - return dev_mem, dev_mem_used + return _get_device_memory_info_without_hbm(dev_card_id, dev_device_id) + + +def _get_device_memory_info_without_hbm(dev_card_id, dev_device_id) -> tuple[int, int]: + """ + Get device memory information from the non-HBM calls. + + Args: + dev_card_id: + The card ID of the device. + dev_device_id: + The device ID of the device. + + Returns: + A tuple containing total memory and used memory in MiB. + + """ + try: + dev_memory_info = pydcmi.dcmi_get_device_memory_info_v3( + dev_card_id, + dev_device_id, + ) + except pydcmi.DCMIError: + # An older driver exposes the V2 call only, as the operator's + # MemoryHandler.V2 uses. + dev_memory_info_v2 = pydcmi.dcmi_get_device_memory_info_v2( + dev_card_id, + dev_device_id, + ) + dev_mem = dev_memory_info_v2.memory_size + # Divergence from the operator, deliberate: it computes + # `memory_size - memory_available` here too, but the V2 struct has no + # available figure at all and its conversion leaves that field zero, + # so it reports every card as fully used -- indistinguishable from a + # real out-of-memory condition. The utilization percentage the struct + # does carry is the only used-memory signal on this path. + return dev_mem, dev_mem * dev_memory_info_v2.utiliza // 100 + + return ( + dev_memory_info.memory_size, + dev_memory_info.memory_size - dev_memory_info.memory_available, + ) def _get_device_memory_status(dev_card_id, dev_device_id) -> DeviceMemoryStatusEnum: @@ -450,34 +656,6 @@ def _get_device_roce_network_info( return ip, mask, gateway -def _get_device_virtual_info( - dev_card_id, - dev_device_id, -) -> pydcmi.c_dcmi_vdev_query_stru | None: - """ - Get device virtual information. - - Returns: - A c_dcmi_vdev_query_stru object if successful, None otherwise. - - """ - try: - c_vdev_query_stru = pydcmi.c_dcmi_vdev_query_stru() - pydcmi.dcmi_get_device_info( - dev_card_id, - dev_device_id, - pydcmi.DCMI_MAIN_CMD_VDEV_MNG, - pydcmi.DCMI_VMNG_SUB_CMD_GET_VDEV_RESOURCE, - c_vdev_query_stru, - ) - except pydcmi.DCMIError: - debug_log_exception(logger, "Failed to get device virtual info") - else: - return c_vdev_query_stru - - return None - - def _get_toolkit_home() -> Path: """ Resolve the Ascend toolkit home directory. diff --git a/gpustack_runtime/detector/cambricon.py b/gpustack_runtime/detector/cambricon.py index 958077f..b1a5f0a 100644 --- a/gpustack_runtime/detector/cambricon.py +++ b/gpustack_runtime/detector/cambricon.py @@ -1,29 +1,48 @@ from __future__ import annotations as __future_annotations__ -import json +import contextlib import logging +import re from functools import lru_cache +from pathlib import Path from .. import envs -from ..logging import debug_log_exception -from . import DeviceMemoryStatusEnum -from .__types__ import Detector, Device, Devices, ManufacturerEnum +from ..logging import debug_log_exception, debug_log_warning +from . import pycndev +from .__types__ import ( + Detector, + Device, + DeviceMemoryStatusEnum, + Devices, + ManufacturerEnum, + merge_devices_usage, +) from .__utils__ import ( PCIDevice, - execute_shell_command, + get_brief_version, + get_numa_node_by_bdf, get_pci_devices, get_utilization, - safe_float, - safe_int, - support_command, ) logger = logging.getLogger(__name__) +_NEUWARE_VERSION_PATH = Path("/usr/local/neuware/version.txt") +""" +Where the CNToolkit package records the Neuware version, i.e. the runtime +version. The single path the operator's getRuntimeVersion stats. +""" + +_NEUWARE_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") +""" +The version pattern the operator matches within that file, so a decorated line +still yields the version alone. +""" + class CambriconDetector(Detector): """ - Detect Cambricon GPUs. + Detect Cambricon MLUs. """ @staticmethod @@ -46,7 +65,11 @@ def is_supported() -> bool: logger.debug("No Cambricon PCI devices found") return supported - supported = support_command("cnmon") + try: + pycndev.cndevInit() + supported = True + except Exception: + debug_log_exception(logger, "Failed to initialize CNDev") return supported @@ -62,12 +85,12 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.CAMBRICON) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Cambricon GPUs using cnmon tool. + Detect Cambricon MLUs' inventory using pycndev, without usage metrics. Returns: - A list of detected Cambricon GPU devices, + A list of detected Cambricon MLU devices, or None if not supported. Raises: @@ -80,54 +103,246 @@ def detect(self) -> Devices | None: ret: Devices = [] try: - output = execute_shell_command( - "cnmon info -e -m -u -j > /dev/null && cat cnmon_info.json", - ) - """ - Example output: - TODO(thxCode): Add example output here. - """ - - output_json = json.loads(output) - dev_infos = output_json.get("CnmonInfo", []) - for dev_info in dev_infos: - dev_index = safe_int(dev_info.get("CardNum")) - dev_name = dev_info.get("ProductName") - dev_uuid = dev_info.get("UUID") - - dev_util_info = dev_info.get("Utilization", {}) - dev_cores_util = safe_float(dev_util_info.get("MLUAverage", 0)) - - dev_mem_usage_info = dev_info.get("PhysicalMemUsage", {}) - dev_mem = safe_int(dev_mem_usage_info.get("Total", 0)) - dev_mem_used = safe_int(dev_mem_usage_info.get("Used", 0)) - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - - dev_temp_info = dev_info.get("Temperature", {}) - dev_temp = safe_float(dev_temp_info.get("Chip", 0)) + pycndev.cndevInit() + + # The Neuware version is a host-wide file read, not a per-device + # driver call, so it is read once above the loop, as the operator's + # DetectAccelerator does. + sys_runtime_ver_original = _get_runtime_version() + sys_runtime_ver = get_brief_version(sys_runtime_ver_original) + + dev_count = pycndev.cndevGetDeviceCount() + dev_skipped_error = None + dev_skipped = 0 + for dev_idx in range(dev_count): + try: + dev = pycndev.cndevGetDeviceHandleByIndex(dev_idx) + dev_uuid = pycndev.cndevGetUUID(dev) + dev_name = pycndev.cndevGetCardNameByDevId(dev) + dev_mem_info = pycndev.cndevGetMemoryUsageV2(dev) + dev_bdf = pycndev.cndevGetPCIeBusId(dev) + except pycndev.CNDevError as e: + # A card whose identity or inventory cannot be read is + # skipped, as the operator's DetectAccelerator does: one + # faulty card must cost that card, not every card of the + # host. Skipping does not renumber the survivors -- an index + # is what the driver enumerated the card at. + debug_log_warning( + logger, + "Failed to fetch device %d, skipping it", + dev_idx, + ) + dev_skipped_error = e + dev_skipped += 1 + continue + + # cndev.h calls the unit MB, but the operator assigns it straight + # into its MiB memory field, cnmon reports MiB, and this repo's + # Ascend detector treats its own MB-declared sizes the same way. + # Converting would under-report ~4.8% against the operator on the + # same host, which is the very discrepancy Story 1 removes. + dev_mem = dev_mem_info.physicalMemoryTotal + dev_mem_status = _get_memory_status(dev) + + dev_driver_ver = None + with contextlib.suppress(pycndev.CNDevError): + dev_ver_info = pycndev.cndevGetVersionInfo(dev) + # The operator formats major.minor only; the build number + # comes free from the same query and is the digit a driver + # bug is diagnosed by, so it is kept. + dev_driver_ver = ( + f"{dev_ver_info.driverMajorVersion}" + f".{dev_ver_info.driverMinorVersion}" + f".{dev_ver_info.driverBuildVersion}" + ) + + dev_numa = get_numa_node_by_bdf(dev_bdf) + if not dev_numa: + with contextlib.suppress(pycndev.CNDevError): + dev_numa_node_id = pycndev.cndevGetNUMANodeIdByDevId(dev) + if dev_numa_node_id.nodeId >= 0: + dev_numa = str(dev_numa_node_id.nodeId) dev_appendix = { - "vgpu": False, + "bdf": dev_bdf, } + if dev_numa: + dev_appendix["numa"] = dev_numa ret.append( Device( manufacturer=self.manufacturer, - index=dev_index, + index=dev_idx, name=dev_name, uuid=dev_uuid, - cores_utilization=dev_cores_util, + driver_version=dev_driver_ver, + runtime_version=sys_runtime_ver, + runtime_version_original=sys_runtime_ver_original, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, appendix=dev_appendix, ), ) + except pycndev.CNDevError: + debug_log_exception(logger, "Failed to fetch devices") + raise except Exception: debug_log_exception(logger, "Failed to process devices fetching") raise + if dev_skipped and dev_skipped == dev_count: + # Skipping one card of several is the graceful degradation this + # vendor is allowed; skipping every one of them is systemic -- a + # driver not exporting a call the loop needs, say -- and reporting it + # as an empty inventory is indistinguishable from a host that has no + # MLU at all. Fail as the other eight vendors do. + raise dev_skipped_error + return ret + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of Cambricon MLUs using pycndev, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pycndev.cndevInit() + + dev_count = pycndev.cndevGetDeviceCount() + dev_skipped_error = None + dev_skipped = 0 + for dev_idx in range(dev_count): + # CNDev enumerates by index, so the whole set is read and + # merge_devices_usage keeps what the caller asked for, as the + # operator's MonitorAccelerator does. + try: + dev = pycndev.cndevGetDeviceHandleByIndex(dev_idx) + dev_uuid = pycndev.cndevGetUUID(dev) + dev_mem_info = pycndev.cndevGetMemoryUsageV2(dev) + except pycndev.CNDevError as e: + debug_log_warning( + logger, + "Failed to fetch device %d usage, skipping it", + dev_idx, + ) + dev_skipped_error = e + dev_skipped += 1 + continue + + dev_mem = dev_mem_info.physicalMemoryTotal + dev_mem_used = dev_mem_info.physicalMemoryUsed + dev_mem_status = _get_memory_status(dev) + + dev_cores_util = 0 + with contextlib.suppress(pycndev.CNDevError): + dev_util_info = pycndev.cndevGetDeviceUtilizationInfo(dev) + dev_cores_util = dev_util_info.averageCoreUtilization + + dev_temp = None + with contextlib.suppress(pycndev.CNDevError): + dev_temp_info = pycndev.cndevGetTemperatureInfo(dev) + dev_temp = dev_temp_info.chip + + dev_power_used = None + with contextlib.suppress(pycndev.CNDevError): + dev_power_info = pycndev.cndevGetDevicePowerInfo(dev) + dev_power_used = dev_power_info.usage + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pycndev.CNDevError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + if dev_skipped and dev_skipped == dev_count: + # Every card refusing its metrics is systemic, not one faulty card, + # and merging nothing would leave the caller reading the information + # query's zeroes as an idle host. + raise dev_skipped_error + + return merge_devices_usage(devices, usages) + + +def _get_memory_status(device: int) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + The verdict is the card-wide health bit, exactly as the operator derives it + (`memoryUnhealthy = healthInfo.Health == 0`): CNDev reports no per-memory + health, and the ECC counters cndevGetECCInfo exposes take no part in it. + + Args: + device: + The device handle. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pycndev.CNDevError): + dev_health_state = pycndev.cndevGetCardHealthStateV2(device) + if dev_health_state.health == 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + +def _get_runtime_version() -> str | None: + """ + Get the Neuware version installed on the host, i.e. the runtime version. + + Returns: + The Neuware version, or None if it cannot be read. + + """ + with contextlib.suppress(OSError): + content = _NEUWARE_VERSION_PATH.read_text().strip() + if match := _NEUWARE_VERSION_PATTERN.search(content): + return match.group() + + return None diff --git a/gpustack_runtime/detector/hygon.py b/gpustack_runtime/detector/hygon.py index 2bc93d3..fd704b5 100644 --- a/gpustack_runtime/detector/hygon.py +++ b/gpustack_runtime/detector/hygon.py @@ -15,6 +15,7 @@ Devices, ManufacturerEnum, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -23,11 +24,10 @@ get_brief_version, get_numa_node_by_bdf, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) -from .amd import _get_arch_family +from .amd import _get_arch_family, _get_pci_device_name_by_bdf logger = logging.getLogger(__name__) @@ -77,9 +77,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.HYGON) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Hygon GPUs using pyrocmsmi. + Detect Hygon GPUs' inventory using pyrocmsmi, without usage metrics. Returns: A list of detected Hygon GPU devices, @@ -128,7 +128,23 @@ def detect(self) -> Devices | None: hsa_agents.get(dev_bdf) or hsa_agents.get(dev_uuid) or pyhsa.Agent() ) - dev_name = dev_hsa_agent.name + # The operator resolves the name from the local PCI ID database + # first: pci.ids knows the board -- the subsystem vendor's name + # for the card -- where the driver only knows the chip. The + # driver-reported names stay as fallbacks. + dev_name = _get_pci_device_name_by_bdf(dev_bdf) + if not dev_name: + dev_name = dev_hsa_agent.name + if not dev_name and dev_card_id is not None: + # The operator asks libdrm for the board's marketing name + # between the HSA and the driver name, so this is that step. + # Reached only when the two above found nothing, which is + # why the device is opened here rather than up front. + with ( + contextlib.suppress(pyamdgpu.AMDGPUError), + pyamdgpu.amdgpu_device(dev_card_id) as dev_gpudev, + ): + dev_name = pyamdgpu.amdgpu_get_marketing_name(dev_gpudev) if not dev_name: dev_name = pyrocmsmi.rsmi_dev_name_get(dev_idx) @@ -152,22 +168,9 @@ def detect(self) -> Devices | None: if not dev_asic_family_id: dev_asic_family_id = dev_gpudev_info.family_id - dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) - dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), - ) dev_mem_status = DeviceMemoryStatusEnum.HEALTHY if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyrocmsmi.ROCMSMIError): @@ -177,10 +180,9 @@ def detect(self) -> Devices | None: if dev_ecc_count.uncorrectable_err > 0: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # The power limit is inventory, while the used power belongs to + # the usage query. dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx) - dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) - - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: @@ -191,7 +193,6 @@ def detect(self) -> Devices | None: dev_appendix = { "arch_family": _get_arch_family(dev_asic_family_id), - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -212,14 +213,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -232,6 +228,89 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Hygon GPUs' usage using pyrocmsmi. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyrocmsmi.rsmi_init() + + devs_count = pyrocmsmi.rsmi_num_monitor_devices() + for dev_idx in range(devs_count): + dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" + + dev_cores_util = pyrocmsmi.rsmi_dev_busy_percent_get(dev_idx) + dev_temp = pyrocmsmi.rsmi_dev_temp_metric_get(dev_idx) + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), + ) + # Health is reported by both queries, as the operator reports it + # from DetectAccelerator and MonitorAccelerator alike. + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyrocmsmi.ROCMSMIError): + dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( + dev_idx, + ) + if dev_ecc_count.uncorrectable_err > 0: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyrocmsmi.ROCMSMIError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between Hygon GPUs. @@ -246,7 +325,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None diff --git a/gpustack_runtime/detector/iluvatar.py b/gpustack_runtime/detector/iluvatar.py index 211fcbe..5689519 100644 --- a/gpustack_runtime/detector/iluvatar.py +++ b/gpustack_runtime/detector/iluvatar.py @@ -15,6 +15,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -24,7 +25,6 @@ get_numa_node_by_bdf, get_numa_nodeset_size, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) @@ -77,9 +77,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.ILUVATAR) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect Iluvatar GPUs using pyixml. + Detect Iluvatar GPUs' inventory using pyixml, without usage metrics. Returns: A list of detected Iluvatar GPU devices, @@ -122,9 +122,15 @@ def detect(self) -> Devices | None: dev = pyixml.nvmlDeviceGetHandleByIndex(dev_idx) dev_index = dev_idx - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - with contextlib.suppress(pyixml.NVMLError): - dev_index = pyixml.nvmlDeviceGetMinorNumber(dev) + + # Device.index is the enumeration index, while the driver's + # minor number is what /dev/iluvatar{N} is made of, so the + # latter goes to the appendix. Mirrors the operator, which + # keeps a sequential Index next to PhysicalIndexes, and omits + # the physical one when the driver cannot answer. + dev_minor_number = None + with contextlib.suppress(pyixml.NVMLError): + dev_minor_number = pyixml.nvmlDeviceGetMinorNumber(dev) dev_name = pyixml.nvmlDeviceGetName(dev) @@ -135,49 +141,25 @@ def detect(self) -> Devices | None: dev_cores = pyixml.nvmlDeviceGetNumGpuCores(dev) dev_mem = 0 - dev_mem_used = 0 dev_mem_status = DeviceMemoryStatusEnum.HEALTHY with contextlib.suppress(pyixml.NVMLError): - dev_mem_info = pyixml.nvmlDeviceGetMemoryInfo(dev) + # Prefer the v2 memory structure, falling back to v1 -- + # mirrors the operator's GetMemoryInfoV, which drops the + # device only when neither call succeeds. + dev_mem_info = _get_memory_info(dev) dev_mem = byte_to_mebibyte( # byte to MiB dev_mem_info.total, ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: with contextlib.suppress(pyixml.NVMLError): dev_health = pyixml.ixmlDeviceGetHealth(dev) if dev_health != pyixml.IXML_HEALTH_OK: dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - dev_cores_util = None - with contextlib.suppress(pyixml.NVMLError): - dev_util_rates = pyixml.nvmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_temp = None - with contextlib.suppress(pyixml.NVMLError): - dev_temp = pyixml.nvmlDeviceGetTemperature( - dev, - pyixml.NVML_TEMPERATURE_GPU, - ) - dev_power = None - dev_power_used = None with contextlib.suppress(pyixml.NVMLError): dev_power = pyixml.nvmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pyixml.nvmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_cc = None with contextlib.suppress(pyixml.NVMLError): @@ -188,8 +170,6 @@ def detect(self) -> Devices | None: dev_pci_info = pyixml.nvmlDeviceGetPciInfo(dev) dev_bdf = str(dev_pci_info.busIdLegacy).lower() - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf - dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: with contextlib.suppress(pyixml.NVMLError): @@ -201,9 +181,10 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_numa: dev_appendix["numa"] = dev_numa @@ -218,14 +199,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -238,6 +214,106 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch Iluvatar GPUs' usage using pyixml. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + usages: Devices = [] + + try: + pyixml.nvmlInit() + + dev_count = pyixml.nvmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pyixml.nvmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pyixml.nvmlDeviceGetUUID(dev) + + dev_mem = 0 + dev_mem_used = 0 + dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + with contextlib.suppress(pyixml.NVMLError): + # Same v2-then-v1 fallback as detect_info: the operator's + # MonitorAccelerator re-reads the memory info rather than + # trusting a previous pass. + dev_mem_info = _get_memory_info(dev) + dev_mem = byte_to_mebibyte( # byte to MiB + dev_mem_info.total, + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + dev_mem_info.used, + ) + if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + with contextlib.suppress(pyixml.NVMLError): + dev_health = pyixml.ixmlDeviceGetHealth(dev) + if dev_health != pyixml.IXML_HEALTH_OK: + dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + + dev_cores_util = None + with contextlib.suppress(pyixml.NVMLError): + dev_util_rates = pyixml.nvmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_temp = None + with contextlib.suppress(pyixml.NVMLError): + dev_temp = pyixml.nvmlDeviceGetTemperature( + dev, + pyixml.NVML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pyixml.NVMLError): + dev_power_used = ( + pyixml.nvmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + except pyixml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -252,7 +328,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -302,3 +378,30 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: raise return ret + + +def _get_memory_info(dev): + """ + Read a device's memory info, preferring the v2 structure with a v1 + fallback. + + Mirrors the operator's GetMemoryInfoV, which tries + ``nvmlDeviceGetMemoryInfo_v2`` before falling back to the v1 call: the + runtime previously read the v1 accessor only, so a driver exposing just + v2 failed here where the operator succeeded. + + Args: + dev: + The device handle. + + Returns: + The memory info structure, from whichever accessor answered. + + Raises: + pyixml.NVMLError: If neither accessor succeeds. + + """ + try: + return pyixml.nvmlDeviceGetMemoryInfo(dev, version=pyixml.nvmlMemory_v2) + except pyixml.NVMLError: + return pyixml.nvmlDeviceGetMemoryInfo(dev) diff --git a/gpustack_runtime/detector/metax.py b/gpustack_runtime/detector/metax.py index ebd581c..55508e1 100644 --- a/gpustack_runtime/detector/metax.py +++ b/gpustack_runtime/detector/metax.py @@ -16,6 +16,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -90,9 +91,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.METAX) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect MetaX GPUs using pymtml. + Detect MetaX GPUs' inventory using pymxsml, without usage metrics. Returns: A list of detected MetaX GPU devices, @@ -122,62 +123,31 @@ def detect(self) -> Devices | None: pymxsml.MXSML_VERSION_DRIVER, ) + # No virtualization-mode filter is applied: whatever the driver + # enumerates is reported. This used to skip + # MXSML_VIRTUALIZATION_MODE_PF, dropping the physical function, + # i.e. the whole card, so a virtualization-enabled host came up + # with zero devices. The operator drops VF instead; reporting + # every mode is a deliberate divergence from it. dev_info = pymxsml.mxSmlGetDeviceInfo(dev_idx) dev_uuid = dev_info.uuid dev_name = dev_info.deviceName - if dev_info.mode == pymxsml.MXSML_VIRTUALIZATION_MODE_PF: - continue - - dev_core_util = pymxsml.mxSmlGetDeviceIpUsage( - dev_idx, - pymxsml.MXSML_USAGE_XCORE, - ) - if dev_core_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_core_util = 0 dev_mem_info = pymxsml.mxSmlGetMemoryInfo(dev_idx) dev_mem = kibibyte_to_mebibyte( # KiB to MiB dev_mem_info.vramTotal, ) - dev_mem_used = kibibyte_to_mebibyte( # KiB to MiB - dev_mem_info.vramUse, - ) - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pymxsml.MXSMLError): - dev_ecc_errors = pymxsml.mxSmlGetTotalEccErrors(dev_idx) - if dev_ecc_errors.dramUE > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_temp = ( - pymxsml.mxSmlGetTemperatureInfo( - dev_idx, - pymxsml.MXSML_TEMPERATURE_HOTSPOT, - ) - // 100 # mC to C - ) + dev_mem_status = _get_memory_status(dev_idx) + # The board power *limit* is inventory, unlike the board power + # *usage*, which the usage query reads. dev_power = ( pymxsml.mxSmlGetBoardPowerLimit(dev_idx) // 1000 # mW to W ) - dev_power_used = None - dev_power_info = pymxsml.mxSmlGetBoardPowerInfo(dev_idx) - if dev_power_info: - dev_power_used = ( - sum(i.power if i.power else 0 for i in dev_power_info) - // 1000 # mW to W - ) dev_bdf = dev_info.bdfId dev_card_id, dev_renderd_id = _get_card_and_renderd_id(dev_bdf) - dev_is_vgpu = dev_info.mode == pymxsml.MXSML_VIRTUALIZATION_MODE_VF - dev_numa = get_numa_node_by_bdf(dev_bdf) if not dev_numa: with contextlib.suppress(pymxsml.MXSMLError): @@ -188,7 +158,6 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -207,14 +176,9 @@ def detect(self) -> Devices | None: driver_version=dev_driver_ver, runtime_version=sys_runtime_ver, runtime_version_original=sys_runtime_ver_original, - cores_utilization=dev_core_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -228,6 +192,103 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of MetaX GPUs using pymxsml, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pymxsml.mxSmlInit() + + dev_count = pymxsml.mxSmlGetDeviceCount() + for dev_idx in range(dev_count): + # MXSML enumerates by index and offers no lookup by UUID, so the + # whole set is read and merge_devices_usage keeps what the caller + # asked for, as the operator's MonitorAccelerator does. + dev_info = pymxsml.mxSmlGetDeviceInfo(dev_idx) + dev_uuid = dev_info.uuid + + dev_core_util = pymxsml.mxSmlGetDeviceIpUsage( + dev_idx, + pymxsml.MXSML_USAGE_XCORE, + ) + if dev_core_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_core_util = 0 + + dev_mem_info = pymxsml.mxSmlGetMemoryInfo(dev_idx) + dev_mem = kibibyte_to_mebibyte( # KiB to MiB + dev_mem_info.vramTotal, + ) + dev_mem_used = kibibyte_to_mebibyte( # KiB to MiB + dev_mem_info.vramUse, + ) + dev_mem_status = _get_memory_status(dev_idx) + + dev_temp = ( + pymxsml.mxSmlGetTemperatureInfo( + dev_idx, + pymxsml.MXSML_TEMPERATURE_HOTSPOT, + ) + // 100 # mC to C + ) + + dev_power_used = None + dev_power_info = pymxsml.mxSmlGetBoardPowerInfo(dev_idx) + if dev_power_info: + dev_power_used = ( + sum(i.power if i.power else 0 for i in dev_power_info) + // 1000 # mW to W + ) + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_core_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pymxsml.MXSMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -242,7 +303,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -293,6 +354,34 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_status(dev_idx: int) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + Args: + dev_idx: + The device index. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pymxsml.MXSMLError): + dev_ecc_errors = pymxsml.mxSmlGetTotalEccErrors(dev_idx) + if dev_ecc_errors.dramUE > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_card_and_renderd_id(dev_bdf: str) -> tuple[int | None, int | None]: """ Get the card ID and renderD ID for a given device bdf. diff --git a/gpustack_runtime/detector/mthreads.py b/gpustack_runtime/detector/mthreads.py index 0680617..0c6a764 100644 --- a/gpustack_runtime/detector/mthreads.py +++ b/gpustack_runtime/detector/mthreads.py @@ -14,6 +14,7 @@ ManufacturerEnum, Topology, TopologyDistanceEnum, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -86,9 +87,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.MTHREADS) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect MThreads GPUs using pymtml. + Detect MThreads GPUs' inventory using pymtml, without usage metrics. Returns: A list of detected MThreads GPU devices, @@ -115,62 +116,34 @@ def detect(self) -> Devices | None: dev_uuid = "" dev_name = "" dev_cores = 0 - dev_power_used = None dev_pci_info = None - dev_is_vgpu = False dev = pymtml.mtmlLibraryInitDeviceByIndex(dev_idx) try: - dev_props = pymtml.mtmlDeviceGetProperty(dev) - dev_is_vgpu = ( - dev_props.virtRole == pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE - ) - if ( - dev_is_vgpu - and dev_props.mpcCap != pymtml.MTML_MPC_TYPE_INSTANCE - ): - continue - + # No virtRole filter is applied: whatever the driver + # enumerates is reported. This used to read the device + # property and skip a MTML_VIRT_ROLE_HOST_VIRTDEVICE card + # whose `mpcCap` was not MTML_MPC_TYPE_INSTANCE -- a field + # c_mtmlDeviceProperty_t does not have, so the read raised + # AttributeError and failed the whole detect pass instead. + # The operator skips GUEST_VIRTDEVICE instead; reporting + # every role is a deliberate divergence from it. dev_uuid = pymtml.mtmlDeviceGetUUID(dev) dev_name = pymtml.mtmlDeviceGetName(dev) dev_cores = pymtml.mtmlDeviceCountGpuCores(dev) - dev_power_used = pymtml.mtmlDeviceGetPowerUsage(dev) dev_pci_info = pymtml.mtmlDeviceGetPciInfo(dev) finally: pymtml.mtmlLibraryFreeDevice(dev) + # MTML binds no power *limit* call, only the power *usage* the + # usage query reads, so the device carries no power limit. + dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY + dev_mem_status = DeviceMemoryStatusEnum.UNKNOWN with pymtml.mtmlMemoryContext(dev) as devmem: dev_mem = byte_to_mebibyte( # byte to MiB pymtml.mtmlMemoryGetTotal(devmem), ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - pymtml.mtmlMemoryGetUsed(devmem), - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pymtml.mtmlMemoryGetEccErrorCounter( - devmem, - pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED, - pymtml.MTML_VOLATILE_ECC, - pymtml.MTML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_cores_util = None - dev_temp = None - with pymtml.mtmlGpuContext(dev) as devgpu: - dev_cores_util = pymtml.mtmlGpuGetUtilization(devgpu) - dev_temp = pymtml.mtmlGpuGetTemperature(devgpu) - - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 + dev_mem_status = _get_memory_status(devmem) dev_bdf = f"{dev_pci_info.segment:04x}:{dev_pci_info.bus:02x}:{dev_pci_info.device:02x}.0" @@ -186,7 +159,6 @@ def detect(self) -> Devices | None: dev_numa = bitmask_to_str(list(dev_node_affinity)) dev_appendix = { - "vgpu": dev_is_vgpu, "bdf": dev_bdf, } if dev_numa: @@ -200,13 +172,8 @@ def detect(self) -> Devices | None: name=dev_name, driver_version=sys_driver_ver, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -223,6 +190,91 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of MThreads GPUs using pymtml, merged into the given + devices in place. + + Args: + devices: + The devices to refresh, matched by UUID. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if devices is None: + return None + + usages: Devices = [] + + try: + pymtml.mtmlLibraryInit() + + dev_count = pymtml.mtmlLibraryCountDevice() + for dev_idx in range(dev_count): + # MTML enumerates by index, so the whole set is read and + # merge_devices_usage keeps what the caller asked for, + # as the operator's MonitorAccelerator does. + dev = pymtml.mtmlLibraryInitDeviceByIndex(dev_idx) + try: + dev_uuid = pymtml.mtmlDeviceGetUUID(dev) + dev_power_used = pymtml.mtmlDeviceGetPowerUsage(dev) + + with pymtml.mtmlMemoryContext(dev) as devmem: + dev_mem = byte_to_mebibyte( # byte to MiB + pymtml.mtmlMemoryGetTotal(devmem), + ) + dev_mem_used = byte_to_mebibyte( # byte to MiB + pymtml.mtmlMemoryGetUsed(devmem), + ) + dev_mem_status = _get_memory_status(devmem) + + with pymtml.mtmlGpuContext(dev) as devgpu: + dev_cores_util = pymtml.mtmlGpuGetUtilization(devgpu) + dev_temp = pymtml.mtmlGpuGetTemperature(devgpu) + finally: + pymtml.mtmlLibraryFreeDevice(dev) + + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + except pymtml.MTMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between MThreads GPUs. @@ -237,7 +289,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -315,6 +367,41 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_status( + devmem: pymtml.c_mtmlMemory_t, +) -> DeviceMemoryStatusEnum: + """ + Get the memory status of a given device. + + Both the information and the usage query report it, mirroring the operator, + which flags a card unhealthy from DetectAccelerator and MonitorAccelerator + alike. The usage query cannot skip it: merging usage overwrites the status, + so a status it did not read would erase the one the information query found. + + Args: + devmem: + The device memory handle. + + Returns: + The memory status of the device. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pymtml.MTMLError): + dev_mem_ecc_errors = pymtml.mtmlMemoryGetEccErrorCounter( + devmem, + pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED, + pymtml.MTML_VOLATILE_ECC, + pymtml.MTML_MEMORY_LOCATION_DRAM, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_links_state( dev: pymtml.c_mtmlDevice_t, ) -> dict | None: diff --git a/gpustack_runtime/detector/nvidia.py b/gpustack_runtime/detector/nvidia.py index 4b5be6f..8ec4a51 100644 --- a/gpustack_runtime/detector/nvidia.py +++ b/gpustack_runtime/detector/nvidia.py @@ -18,6 +18,7 @@ ManufacturerEnum, TopologyDistanceEnum, index_mig_devices, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -81,9 +82,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.NVIDIA) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect NVIDIA GPUs using pynvml. + Detect NVIDIA GPUs' inventory using pynvml, without usage metrics. Returns: A list of detected NVIDIA GPU devices, @@ -99,8 +100,6 @@ def detect(self) -> Devices | None: ret: Devices = [] try: - pci_devs = NVIDIADetector.detect_pci_devices() - pynvml.nvmlInit() sys_driver_ver = pynvml.nvmlSystemGetDriverVersion() @@ -134,6 +133,13 @@ def detect(self) -> Devices | None: dev_cc_t = pynvml.nvmlDeviceGetCudaComputeCapability(dev) dev_cc = ".".join(map(str, dev_cc_t)) + # Unversioned on purpose, unlike the memory query above: pynvml + # exposes no v2 PCI accessor to prefer or fall back to -- + # `nvmlDeviceGetPciInfo` is its alias of `nvmlDeviceGetPciInfo_v3`, + # a superset of the v2 structure the operator's `GetPciInfoV` + # prefers, and `nvmlPciInfo_v2_t` is declared but unused. The + # operator never reads the structure's string either, its + # `GetBusId()` formatting the BDF from domain/bus/device. dev_pci_info = pynvml.nvmlDeviceGetPciInfo(dev) dev_bdf = str(dev_pci_info.busIdLegacy).lower() @@ -147,30 +153,27 @@ def detect(self) -> Devices | None: ) dev_numa = bitmask_to_str(list(dev_node_affinity)) - dev_temp = None - with contextlib.suppress(pynvml.NVMLError): - dev_temp = pynvml.nvmlDeviceGetTemperature( - dev, - pynvml.NVML_TEMPERATURE_GPU, - ) - + # The power limit is inventory; the power actually drawn is + # usage, and belongs to detect_usage. dev_power = None - dev_power_used = None with contextlib.suppress(pynvml.NVMLError): dev_power = pynvml.nvmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pynvml.nvmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_mig_mode = pynvml.NVML_DEVICE_MIG_DISABLE with contextlib.suppress(pynvml.NVMLError): dev_mig_mode, _ = pynvml.nvmlDeviceGetMigMode(dev) dev_index = dev_idx - if envs.GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: - with contextlib.suppress(pynvml.NVMLError): - dev_index = pynvml.nvmlDeviceGetMinorNumber(dev) + + # Device.index is the enumeration index, while the driver's + # minor number is what a device node path is made of, so the + # latter goes to the appendix. Mirrors the operator, which + # keeps a sequential Index next to PhysicalIndexes, and omits + # the physical one when the driver cannot answer. + dev_minor_number = None + with contextlib.suppress(pynvml.NVMLError): + dev_minor_number = pynvml.nvmlDeviceGetMinorNumber(dev) # Report the physical card, whether or not MIG is enabled. # MIG instances are partitioned on demand by the operator's @@ -186,52 +189,34 @@ def detect(self) -> Devices | None: with contextlib.suppress(pynvml.NVMLError): dev_cores = pynvml.nvmlDeviceGetNumGpuCores(dev) - dev_cores_util = _get_sm_util_from_gpm_metrics(dev) - if dev_cores_util is None: - with contextlib.suppress(pynvml.NVMLError): - dev_util_rates = pynvml.nvmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pynvml.NVMLError): - dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(dev) - dev_mem = byte_to_mebibyte( # byte to MiB - dev_mem_info.total, - ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( - dev, - pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, - pynvml.NVML_VOLATILE_ECC, - pynvml.NVML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # Reported as the driver reports it, i.e. what the card can + # actually allocate. A deliberate divergence: the operator adds + # back the ~1/16 that ECC parity carves out of a GDDR part, but + # that capacity is not reachable, and its restored figure is a + # display value that takes no part in allocation. Here `memory` + # does take part, and `memory - memory_used` has to mean free + # space, so restoring it would over-commit every GDDR card with + # ECC enabled. + dev_mem, _ = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pynvml.NVML_VOLATILE_ECC, + pynvml.NVML_MEMORY_LOCATION_DRAM, + ) if dev_mem == 0: - dev_mem, dev_mem_used = get_memory() - - dev_is_vgpu = False - if dev_bdf in pci_devs: - dev_is_vgpu = _is_vgpu(pci_devs[dev_bdf].config) + # A deliberate divergence from the operator, which skips a + # device whose total reads 0: here it falls back to the host + # memory, tolerating WSL and integrated GPUs, which report + # no device memory of their own. + dev_mem, _ = get_memory() dev_appendix = { "arch_family": _get_arch_family(dev_cc_t), - "vgpu": dev_is_vgpu, "mig": dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE: dev_mig_slots = 0 with contextlib.suppress(pynvml.NVMLError): @@ -245,9 +230,7 @@ def detect(self) -> Devices | None: sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf, dev_numa, ) @@ -270,14 +253,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -292,6 +270,120 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch NVIDIA GPUs' usage using pynvml, merged into the given devices. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during fetching. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + # The usage query enumerates the driver's devices on its own and returns + # them keyed by UUID, mirroring the operator's MonitorAccelerator: a + # metrics list is joined by device identity, never by index, as an index + # is not stable across a re-detection. + usages: Devices = [] + + try: + pynvml.nvmlInit() + + dev_count = pynvml.nvmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pynvml.nvmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pynvml.nvmlDeviceGetUUID(dev) + + dev_cores_util = _get_sm_util_from_gpm_metrics(dev) + if dev_cores_util is None: + with contextlib.suppress(pynvml.NVMLError): + dev_util_rates = pynvml.nvmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem, dev_mem_used = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pynvml.NVML_VOLATILE_ECC, + pynvml.NVML_MEMORY_LOCATION_DRAM, + ) + if dev_mem == 0: + # The same deliberate divergence detect_info records: a + # device whose total reads 0 falls back to the host memory. + dev_mem, dev_mem_used = get_memory() + + dev_temp = None + with contextlib.suppress(pynvml.NVMLError): + dev_temp = pynvml.nvmlDeviceGetTemperature( + dev, + pynvml.NVML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pynvml.NVMLError): + dev_power_used = ( + pynvml.nvmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + dev_mig_mode = pynvml.NVML_DEVICE_MIG_DISABLE + with contextlib.suppress(pynvml.NVMLError): + dev_mig_mode, _ = pynvml.nvmlDeviceGetMigMode(dev) + if dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE: + dev_mig_slots = 0 + with contextlib.suppress(pynvml.NVMLError): + dev_mig_slots = pynvml.nvmlDeviceGetMaxMigDeviceCount(dev) + usages.extend( + _get_mig_usages( + dev, + dev_mig_slots, + dev_temp, + dev_power_used, + ), + ) + except pynvml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -306,7 +398,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -390,6 +482,86 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_info( + dev: pynvml.c_nvmlDevice_t, +) -> tuple[int, int]: + """ + Get a device's total and used memory, preferring the v2 memory structure + with a v1 fallback, as the operator's `GetMemoryInfoV` does. + + Args: + dev: + The NVML device handle. + + Returns: + The total and used memory in MiB, both 0 if unreadable. + + """ + # `version` is the packed struct version the driver validates, not a plain + # ordinal, so it is the binding's own constant. Probing for it keeps a + # binding predating the v2 structure on the v1 path, instead of raising the + # dependency floor for it. + dev_mem_info_ver = getattr(pynvml, "nvmlMemory_v2", None) + if dev_mem_info_ver is not None: + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo( + dev, + version=dev_mem_info_ver, + ) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(dev) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + return 0, 0 + + +def _get_memory_status( + dev: pynvml.c_nvmlDevice_t, + ecc_counter_type: int, + memory_location: int, +) -> DeviceMemoryStatusEnum: + """ + Get a device's memory health from its uncorrected ECC error counter. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. + + Args: + dev: + The NVML device handle. + ecc_counter_type: + The ECC counter type to read, volatile or aggregate. + memory_location: + The memory location to read the counter of. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pynvml.NVMLError): + dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( + dev, + pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, + ecc_counter_type, + memory_location, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_gpm_metrics( metrics: list[int], dev: pynvml.c_nvmlDevice_t, @@ -603,54 +775,48 @@ def _get_mig_devices( sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf: str, dev_numa, ) -> list[dict]: """ - Enumerate the card's current MIG devices with the same detail a plain - device carries (profile name, uuid, compute/memory utilization, memory - health, temperature and power), returned as appendix entries of the - physical card rather than standalone devices. Empty when MIG is enabled - but no GPU instances exist yet. + Enumerate the card's current MIG devices with the same inventory detail a + plain device carries (profile name, uuid, cores, total memory and memory + health), returned as appendix entries of the physical card rather than + standalone devices. Empty when MIG is enabled but no GPU instances exist + yet. + + An entry keeps a Device's shape, so the fields the usage query owns are + present at a Device's defaults: `_get_mig_usages` fills them. Each entry's `index` is the driver slot the MIG device was found at: index_mig_devices turns it into the device index once every card is detected. """ ret: list[dict] = [] - with contextlib.suppress(pynvml.NVMLError): - for mdev_idx in range(dev_mig_slots): - mdev = None - with contextlib.suppress(pynvml.NVMLError): - mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing a read + # used to abort the loop, so every later instance vanished from the + # inventory. An empty slot raises here as well, which is how it is + # skipped. + with contextlib.suppress(pynvml.NVMLError): + mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) if not mdev: continue mdev_uuid = pynvml.nvmlDeviceGetUUID(mdev) - mdev_mem = 0 - mdev_mem_used = 0 - mdev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pynvml.NVMLError): - mdev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(mdev) - mdev_mem = byte_to_mebibyte(mdev_mem_info.total) - mdev_mem_used = byte_to_mebibyte(mdev_mem_info.used) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - mdev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( - mdev, - pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, - pynvml.NVML_AGGREGATE_ECC, - pynvml.NVML_MEMORY_LOCATION_SRAM, - ) - if mdev_mem_ecc_errors > 0: - mdev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # A MIG device reports the partition's size, which carries no ECC + # reserve to restore: MIG-capable cards are HBM parts. + mdev_mem, _ = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pynvml.NVML_AGGREGATE_ECC, + pynvml.NVML_MEMORY_LOCATION_SRAM, + ) mdev_appendix = { "arch_family": _get_arch_family(dev_cc_t), - "vgpu": True, "sliced": True, "mig": True, "bdf": dev_bdf, @@ -663,8 +829,6 @@ def _get_mig_devices( mdev_ci_id = pynvml.nvmlDeviceGetComputeInstanceId(mdev) mdev_appendix["compute_instance_id"] = mdev_ci_id - mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) - mdev_name = "" mdev_cores = None mdev_gi = pynvml.nvmlDeviceGetGpuInstanceById(dev, mdev_gi_id) @@ -726,20 +890,84 @@ def _get_mig_devices( "runtime_version_original": sys_runtime_ver_original, "compute_capability": dev_cc, "cores": mdev_cores, - "cores_utilization": mdev_cores_util, + "cores_utilization": 0, "memory": mdev_mem, - "memory_used": mdev_mem_used, - "memory_utilization": get_utilization(mdev_mem_used, mdev_mem), + "memory_used": 0, + "memory_utilization": 0, "memory_status": mdev_mem_status, - "temperature": dev_temp, + "temperature": None, "power": dev_power, - "power_used": dev_power_used, + "power_used": None, "appendix": mdev_appendix, }, ) return ret +def _get_mig_usages( + dev, + dev_mig_slots: int, + dev_temp, + dev_power_used, +) -> Devices: + """ + Fetch the usage of the card's current MIG devices, one UUID-keyed entry per + instance, to merge into the card's `appendix["mig_devices"]`. + + Args: + dev: + The NVML device handle of the card hosting them. + dev_mig_slots: + The number of MIG devices the card can host. + dev_temp: + The card's temperature. + dev_power_used: + The card's used power. + + Returns: + The MIG devices' usage, keyed by UUID. + + """ + ret: Devices = [] + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing its UUID + # or its GPU instance id used to abort the loop, so every later instance + # kept the inventory's defaults -- 0 % and 0 MiB, reported idle while it + # may be running a workload. An empty slot raises here as well, which is + # how it is skipped. + with contextlib.suppress(pynvml.NVMLError): + mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + if not mdev: + continue + + mdev_uuid = pynvml.nvmlDeviceGetUUID(mdev) + + mdev_mem, mdev_mem_used = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pynvml.NVML_AGGREGATE_ECC, + pynvml.NVML_MEMORY_LOCATION_SRAM, + ) + + mdev_gi_id = pynvml.nvmlDeviceGetGpuInstanceId(mdev) + mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) + + ret.append( + Device( + uuid=mdev_uuid, + cores_utilization=mdev_cores_util, + memory_used=mdev_mem_used, + memory_utilization=get_utilization(mdev_mem_used, mdev_mem), + memory_status=mdev_mem_status, + # A MIG device reports neither temperature nor power, so it + # carries the card's. + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + return ret + + def _get_arch_family(dev_cc_t: list[int]) -> str: """ Get the architecture family based on the CUDA compute capability. @@ -774,38 +1002,3 @@ def _get_arch_family(dev_cc_t: list[int]) -> str: case 10 | 12: return "Blackwell" return "Unknown" - - -def _is_vgpu(dev_config: bytes) -> bool: - """ - Determine if the device is a vGPU based on its PCI configuration space. - - """ - status = 0x06 - cap_supported = 0x10 - cap_start = 0x34 - cap_vendor_specific_id = 0x09 - - if dev_config[status] & cap_supported == 0: - return False - - # Find the capability list - dev_cap: bytes | None = None - visited = set() - pos = dev_config[cap_start] - while pos != 0 and pos not in visited and pos < len(dev_config) - 2: - visited.add(pos) - ptr = dev_config[pos : pos + 3] # id, next, length - if ptr[0] == 0xFF: - break - if ptr[0] == cap_vendor_specific_id: - dev_cap = dev_config[pos : pos + ptr[2]] - break - pos = ptr[1] - - if not dev_cap or len(dev_cap) < 5: - return False - - # Check for vGPU signature, - # which is either 0x56 (NVIDIA vGPU) or 0x46 (NVIDIA GRID). - return dev_cap[3] == 0x56 or dev_cap[4] == 0x46 diff --git a/gpustack_runtime/detector/pyamdgpu/__init__.py b/gpustack_runtime/detector/pyamdgpu/__init__.py index a1ddba6..afc2f80 100644 --- a/gpustack_runtime/detector/pyamdgpu/__init__.py +++ b/gpustack_runtime/detector/pyamdgpu/__init__.py @@ -284,3 +284,16 @@ def amdgpu_query_gpu_info(device): ret = fn(device, byref(c_info)) _amdgpuCheckReturn(ret) return c_info + + +def amdgpu_get_marketing_name(device): + fn = _amdgpuGetFunctionPointer("amdgpu_get_marketing_name") + # This one answers with a pointer into libdrm's own table rather than + # filling a buffer, so the return type has to be declared before the call + # or ctypes truncates the pointer to an int. + fn.restype = c_char_p + c_name = fn(device) + # The table carries no entry for every device id, and libdrm answers NULL + # rather than failing, so an unnamed board is empty here and the caller + # falls through to its next name source. + return c_name.decode() if c_name else "" diff --git a/gpustack_runtime/detector/pycndev/__init__.py b/gpustack_runtime/detector/pycndev/__init__.py new file mode 100644 index 0000000..b0363fc --- /dev/null +++ b/gpustack_runtime/detector/pycndev/__init__.py @@ -0,0 +1,840 @@ +## +# Python bindings for the CNDev library +# +# Derived from the Cambricon MLU driver's cndev.h, as vendored by the operator in +# binding/cndev -- the header of record. Only what the Cambricon detector needs is +# bound: device enumeration, identity, inventory and the usage metrics. MLU-Link, +# sMLU/MIM partitioning, per-process accounting and topology are deliberately absent. +## +from __future__ import annotations as __future_annotations__ + +import os +import string +import sys +import threading +from ctypes import * +from functools import wraps +from typing import ClassVar + +## C Type mappings ## +# cndevDevice_t is an __int32_t handle, not an opaque pointer. +cndevDevice_t = c_int32 + +## Constants ## +CNDEV_VERSION_1 = 1 +CNDEV_VERSION_2 = 2 +CNDEV_VERSION_3 = 3 +CNDEV_VERSION_4 = 4 +CNDEV_VERSION_5 = 5 +CNDEV_VERSION_6 = 6 +CNDEV_UUID_SIZE = 37 +CNDEV_ERR_MSG_LENGTH = 512 +CNDEV_HEALTH_SYSTEM_MAX_INCIDENTS = 64 +# The header hardcodes these array lengths in the struct declarations themselves. +CNDEV_TEMPERATURE_CLUSTER_COUNT = 20 +CNDEV_TEMPERATURE_MEMORY_DIE_COUNT = 8 +CNDEV_UTILIZATION_CORE_COUNT = 80 + +## Enums ## +# A C enum here is int-sized: every enumerator in cndev.h is a small non-negative +# value, so the compiler lays them out as a 4-byte signed int. +_cndevNameEnum_t = c_int32 +CNDEV_DEVICE_TYPE_MLU100 = 0 +CNDEV_DEVICE_TYPE_MLU270 = 1 +CNDEV_DEVICE_TYPE_MLU220_M2 = 16 +CNDEV_DEVICE_TYPE_MLU220_EDGE = 17 +CNDEV_DEVICE_TYPE_MLU220_EVB = 18 +CNDEV_DEVICE_TYPE_MLU220_M2i = 19 +CNDEV_DEVICE_TYPE_MLU290 = 20 +CNDEV_DEVICE_TYPE_MLU370 = 23 +CNDEV_DEVICE_TYPE_MLU365 = 24 +CNDEV_DEVICE_TYPE_CE3226 = 25 +CNDEV_DEVICE_TYPE_MLU590 = 26 +CNDEV_DEVICE_TYPE_MLU585 = 27 +CNDEV_DEVICE_TYPE_1V_2201 = 29 +CNDEV_DEVICE_TYPE_MLU580 = 30 +CNDEV_DEVICE_TYPE_MLU570 = 31 +CNDEV_DEVICE_TYPE_1V_2202 = 32 + +## Enums ## +_cndevEnableStatusEnum_t = c_int32 +CNDEV_FEATURE_DISABLED = 0 +CNDEV_FEATURE_ENABLED = 1 + +## Enums ## +_cndevHealthStateEnum_t = c_int32 +CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM = 0 +CNDEV_HEALTH_STATE_DEVICE_GOOD = 1 + +## Enums ## +_cndevDriverHealthStateEnum_t = c_int32 +CNDEV_HEALTH_STATE_DRIVER_EARLY_INITED = 0 +CNDEV_HEALTH_STATE_DRIVER_BRING_UP = 1 +CNDEV_HEALTH_STATE_DRIVER_BOOTING = 2 +CNDEV_HEALTH_STATE_DRIVER_LATEINIT = 3 +CNDEV_HEALTH_STATE_DRIVER_RUNNING = 4 +CNDEV_HEALTH_STATE_DRIVER_BOOT_ERROR = 5 +CNDEV_HEALTH_STATE_DRIVER_RESET = 6 +CNDEV_HEALTH_STATE_DRIVER_RESET_ERROR = 7 +CNDEV_HEALTH_STATE_DRIVER_UNKNOWN = 8 + +## Enums ## +_cndevHealthResult_t = c_int32 +CNDEV_HEALTH_RESULT_PASS = 0 +CNDEV_HEALTH_RESULT_WARN = 10 +CNDEV_HEALTH_RESULT_FAIL = 20 + +## Enums ## +# The incident systems (cndevHealthSystem_t) and diagnosis codes (cndevHealthError_t) +# are only needed as field widths: nothing on the detect path names an incident, so +# their enumerators are left unbound rather than copied and left to rot. +_cndevHealthSystem_t = c_int32 +_cndevHealthError_t = c_int32 + + +## Error Codes ## +CNDEV_SUCCESS = 0 +CNDEV_ERROR_NO_DRIVER = 1 +CNDEV_ERROR_LOW_DRIVER_VERSION = 2 +CNDEV_ERROR_UNSUPPORTED_API_VERSION = 3 +CNDEV_ERROR_UNINITIALIZED = 4 +CNDEV_ERROR_INVALID_ARGUMENT = 5 +CNDEV_ERROR_INVALID_DEVICE_ID = 6 +CNDEV_ERROR_UNKNOWN = 7 +CNDEV_ERROR_MALLOC = 8 +CNDEV_ERROR_INSUFFICIENT_SPACE = 9 +CNDEV_ERROR_NOT_SUPPORTED = 10 +CNDEV_ERROR_INVALID_LINK = 11 +CNDEV_ERROR_NO_DEVICES = 12 +CNDEV_ERROR_NO_PERMISSION = 13 +CNDEV_ERROR_NOT_FOUND = 14 +CNDEV_ERROR_IN_USE = 15 +CNDEV_ERROR_DUPLICATE = 16 +CNDEV_ERROR_TIMEOUT = 17 +CNDEV_ERROR_IN_PROBLEM = 18 +# Not in the header: the operator's binding/cndev/library.go defines the same two codes, +# at the same values, for the failures that happen before any driver call -- the shared +# object is absent, or the symbol is missing from it. +CNDEV_ERROR_FUNCTION_NOT_FOUND = -99998 +CNDEV_ERROR_LIBRARY_NOT_FOUND = -99999 + +## Lib loading ## +cndevLib = None +# Reentrant, unlike the sibling bindings' lock: cndevInit holds it across the whole +# check-then-initialize sequence, and _cndevGetFunctionPointer takes it as well. +libLoadLock = threading.RLock() +_libInitialized = False +_libInitializedException = None + + +## Error Checking ## +class CNDevError(Exception): + _valClassMapping: ClassVar[dict] = {} + + _errcode_to_string: ClassVar[dict] = { + CNDEV_ERROR_NO_DRIVER: "No Driver", + CNDEV_ERROR_LOW_DRIVER_VERSION: "Low Driver Version", + CNDEV_ERROR_UNSUPPORTED_API_VERSION: "Unsupported API Version", + CNDEV_ERROR_UNINITIALIZED: "Library Not Initialized", + CNDEV_ERROR_INVALID_ARGUMENT: "Invalid Argument", + CNDEV_ERROR_INVALID_DEVICE_ID: "Invalid Device ID", + CNDEV_ERROR_UNKNOWN: "Unknown Error", + CNDEV_ERROR_MALLOC: "Memory Allocation Failed", + CNDEV_ERROR_INSUFFICIENT_SPACE: "Insufficient Space", + CNDEV_ERROR_NOT_SUPPORTED: "Not Supported", + CNDEV_ERROR_INVALID_LINK: "Invalid Link", + CNDEV_ERROR_NO_DEVICES: "No Devices", + CNDEV_ERROR_NO_PERMISSION: "No Permission", + CNDEV_ERROR_NOT_FOUND: "Not Found", + CNDEV_ERROR_IN_USE: "In Use", + CNDEV_ERROR_DUPLICATE: "Duplicate", + CNDEV_ERROR_TIMEOUT: "Time Out", + CNDEV_ERROR_IN_PROBLEM: "In Problem", + CNDEV_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", + CNDEV_ERROR_LIBRARY_NOT_FOUND: "Library Not Found", + } + + def __new__(cls, value): + """ + Maps value to a proper subclass of CNDevError. + See _extractCNDevErrorsAsClasses function for more details. + """ + if cls == CNDevError: + cls = CNDevError._valClassMapping.get(value, cls) + obj = Exception.__new__(cls) + obj.value = value + return obj + + def __str__(self): + return CNDevError._errcode_to_string.get( + self.value, + f"Unknown CNDev Error {self.value}", + ) + + def __eq__(self, other): + if isinstance(other, CNDevError): + return self.value == other.value + if isinstance(other, int): + return self.value == other + return False + + +def cndevExceptionClass(cndevErrorCode): + if cndevErrorCode not in CNDevError._valClassMapping: + msg = f"CNDev error code {cndevErrorCode} is not valid" + raise ValueError(msg) + return CNDevError._valClassMapping[cndevErrorCode] + + +def _extractCNDevErrorsAsClasses(): + """ + Generates a hierarchy of classes on top of CNDevError class. + + Each CNDev Error gets a new CNDevError subclass. This way try,except blocks can + filter appropriate exceptions more easily. + + CNDevError is a parent class. Each CNDEV_ERROR_* gets it's own subclass. + e.g. CNDEV_ERROR_INVALID_ARGUMENT will be turned into CNDevError_InvalidArgument. + """ + this_module = sys.modules[__name__] + cndevErrorsNames = [x for x in dir(this_module) if x.startswith("CNDEV_ERROR_")] + for err_name in cndevErrorsNames: + # e.g. Turn CNDEV_ERROR_INVALID_ARGUMENT into CNDevError_InvalidArgument + class_name = "CNDevError_" + string.capwords( + err_name.replace("CNDEV_ERROR_", ""), + "_", + ).replace("_", "") + err_val = getattr(this_module, err_name) + + def gen_new(val): + def new(typ, *args): + obj = CNDevError.__new__(typ, val) + return obj + + return new + + new_error_class = type(class_name, (CNDevError,), {"__new__": gen_new(err_val)}) + new_error_class.__module__ = __name__ + setattr(this_module, class_name, new_error_class) + CNDevError._valClassMapping[err_val] = new_error_class + + +_extractCNDevErrorsAsClasses() + + +def _cndevCheckReturn(ret): + if ret != CNDEV_SUCCESS: + raise CNDevError(ret) + return ret + + +## Function access ## +_cndevGetFunctionPointer_cache = {} + + +def _cndevGetFunctionPointer(name): + global cndevLib + + if name in _cndevGetFunctionPointer_cache: + return _cndevGetFunctionPointer_cache[name] + + libLoadLock.acquire() + try: + if cndevLib is None: + raise CNDevError(CNDEV_ERROR_UNINITIALIZED) + try: + _cndevGetFunctionPointer_cache[name] = getattr(cndevLib, name) + return _cndevGetFunctionPointer_cache[name] + except AttributeError: + raise CNDevError(CNDEV_ERROR_FUNCTION_NOT_FOUND) + finally: + libLoadLock.release() + + +## Structure definitions ## +class _PrintableStructure(Structure): + """ + Abstract class that produces nicer __str__ output than ctypes.Structure. + """ + + _fmt_ = {} + + def __str__(self): + result = [] + for x in self._fields_: + key = x[0] + value = getattr(self, key) + fmt = "%s" + if key in self._fmt_: + fmt = self._fmt_[key] + elif "" in self._fmt_: + fmt = self._fmt_[""] + result.append(("%s: " + fmt) % (key, value)) + return self.__class__.__name__ + "(" + ", ".join(result) + ")" + + def __getattribute__(self, name): + res = super().__getattribute__(name) + if isinstance(res, bytes): + return res.decode() + return res + + def __setattr__(self, name, value): + if isinstance(value, str): + value = value.encode() + super().__setattr__(name, value) + + +class c_cndevCardInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("number", c_uint32), + ] + + +class c_cndevUUID_t(_PrintableStructure): + # uuid is a NUL-terminated __uint8_t array in the header; c_char keeps the same + # layout and reads back as the string the driver wrote. + _fields_: ClassVar = [ + ("version", c_int32), + ("uuid", c_char * CNDEV_UUID_SIZE), + ("ncsUUID64", c_uint64), + ] + + +class c_cndevMemoryInfoV2_t(_PrintableStructure): + # Sizes are MB, and this struct carries no version field. + _fields_: ClassVar = [ + ("physicalMemoryTotal", c_int64), + ("physicalMemoryUsed", c_int64), + ("reservedMemory", c_int64), + ("virtualMemoryTotal", c_int64), + ("virtualMemoryUsed", c_int64), + ("globalMemory", c_uint64), + ("reserved", c_uint64 * 16), + ] + + +class c_cndevVersionInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("mcuMajorVersion", c_uint32), + ("mcuMinorVersion", c_uint32), + ("mcuBuildVersion", c_uint32), + ("driverMajorVersion", c_uint32), + ("driverMinorVersion", c_uint32), + ("driverBuildVersion", c_uint32), + ] + + +class c_cndevECCInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("oneBitError", c_uint64), + ("multipleOneError", c_uint64), + ("multipleError", c_uint64), + ("multipleMultipleError", c_uint64), + ("correctedError", c_uint64), + ("uncorrectedError", c_uint64), + ("totalError", c_uint64), + ("addressForbiddenError", c_uint64), + ] + + +class c_cndevDevicePowerInfo_t(_PrintableStructure): + # Watts, and this struct carries no version field. + _fields_: ClassVar = [ + ("usage", c_int32), + ("cap", c_int32), + ("machine", c_int32), + ("tdp", c_int32), + ("maxPower", c_int32), + ("reserved", c_int32 * 16), + ] + + +class c_cndevTemperatureInfo_t(_PrintableStructure): + # Degrees Celsius. + _fields_: ClassVar = [ + ("version", c_int32), + ("board", c_int32), + ("cluster", c_int32 * CNDEV_TEMPERATURE_CLUSTER_COUNT), + ("memoryDie", c_int32 * CNDEV_TEMPERATURE_MEMORY_DIE_COUNT), + ("chip", c_int32), + ("airInlet", c_int32), + ("airOutlet", c_int32), + ("memory", c_int32), + ("videoInput", c_int32), + ("cpu", c_int32), + ("isp", c_int32), + ] + + +class c_cndevUtilizationInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("averageCoreUtilization", c_int32), + ("coreUtilization", c_int32 * CNDEV_UTILIZATION_CORE_COUNT), + ] + + +class c_cndevCardName_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("id", _cndevNameEnum_t), + ] + + +class c_cndevPCIeInfoV2_t(_PrintableStructure): + # This struct carries no version field. + _fields_: ClassVar = [ + ("subsystemId", c_uint32), + ("deviceId", c_uint32), + ("vendor", c_uint16), + ("subsystemVendor", c_uint16), + ("domain", c_uint32), + ("bus", c_uint32), + ("device", c_uint32), + ("function", c_uint32), + ("moduleId", c_uint16), + ("slotId", c_uint16), + ("reserved", c_uint32 * 8), + ] + + +class c_cndevCardHealthState_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("health", c_int32), + ("deviceState", _cndevHealthStateEnum_t), + ("driverState", _cndevDriverHealthStateEnum_t), + ] + + +class c_cndevDiagErrorDetail_t(_PrintableStructure): + _fields_: ClassVar = [ + ("msg", c_char * CNDEV_ERR_MSG_LENGTH), + ("device_id", c_uint32), + ("bdf", c_uint32), + ("code", _cndevHealthError_t), + ("category", c_uint32), + ("severity", c_uint32), + ] + + +class c_cndevIncidentInfo_t(_PrintableStructure): + _fields_: ClassVar = [ + ("system", _cndevHealthSystem_t), + ("health", _cndevHealthResult_t), + ("error", c_cndevDiagErrorDetail_t), + ] + + +class c_cndevCardHealthStateV2_t(_PrintableStructure): + # This struct carries no version field, and the driver fills all 64 incident slots + # in place: a short definition here is a buffer overrun, not a missing field. + _fields_: ClassVar = [ + ("health", c_int32), + ("deviceState", _cndevHealthStateEnum_t), + ("driverState", _cndevDriverHealthStateEnum_t), + ("overallHealth", _cndevHealthResult_t), + ("incident_count", c_uint32), + ("incidents", c_cndevIncidentInfo_t * CNDEV_HEALTH_SYSTEM_MAX_INCIDENTS), + ("reserved", c_uint32 * 8), + ] + + +class c_cndevNUMANodeId_t(_PrintableStructure): + _fields_: ClassVar = [ + ("version", c_int32), + ("nodeId", c_int32), + ] + + +## Device name fallback table ## +# Mirrors the switch in the operator's binding/cndev/library_device.go, used only when +# the driver exports neither cndevGetCardNameStringByDevId nor cndevGetCardNameString. +# The MLU220 variants all report as the family, and anything unlisted reports as "MLU". +_cndevCardNames = { + CNDEV_DEVICE_TYPE_MLU100: "MLU100", + CNDEV_DEVICE_TYPE_MLU270: "MLU270", + CNDEV_DEVICE_TYPE_MLU220_M2: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_EDGE: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_EVB: "MLU220", + CNDEV_DEVICE_TYPE_MLU220_M2i: "MLU220", + CNDEV_DEVICE_TYPE_MLU290: "MLU290", + CNDEV_DEVICE_TYPE_MLU370: "MLU370", + CNDEV_DEVICE_TYPE_MLU365: "MLU365", + CNDEV_DEVICE_TYPE_CE3226: "CE3226", + CNDEV_DEVICE_TYPE_MLU590: "MLU590", + CNDEV_DEVICE_TYPE_MLU585: "MLU585", + CNDEV_DEVICE_TYPE_MLU580: "MLU580", + CNDEV_DEVICE_TYPE_MLU570: "MLU570", +} + + +## string/bytes conversion for ease of use +def convertStrBytes(func): + @wraps(func) + def wrapper(*args, **kwargs): + # encoding a str returns bytes in python 2 and 3 + args = [arg.encode() if isinstance(arg, str) else arg for arg in args] + res = func(*args, **kwargs) + # In python 2, str and bytes are the same + # In python 3, str is unicode and should be decoded. + # Ctypes handles most conversions, this only effects c_char and char arrays. + if isinstance(res, bytes): + if isinstance(res, str): + return res + return res.decode() + return res + + return wrapper + + +def _LoadCndevLibrary(): + """ + Load the library if it isn't loaded already. + """ + global cndevLib + + if cndevLib is None: + # lock to ensure only one caller loads the library + libLoadLock.acquire() + + try: + # ensure the library still isn't loaded + if cndevLib is None: + if sys.platform.startswith("win"): + # Do not support Windows yet. + raise CNDevError(CNDEV_ERROR_LIBRARY_NOT_FOUND) + # Linux path, + # mirroring the search order of the operator's binding/cndev/library.go: + # the bare soname first, so a driver on the loader's search path wins, + # then the Neuware home, where the CNToolkit package installs it. + neuware_home = os.getenv("NEUWARE_HOME") or "/usr/local/neuware" + locs = [ + "libcndev.so", + os.path.join(neuware_home, "lib64", "libcndev.so"), + os.path.join(neuware_home, "lib", "libcndev.so"), + ] + load_error = None + for loc in locs: + try: + cndevLib = CDLL(loc) + break + except OSError as e: + load_error = e + if cndevLib is None: + # Chain the loader's own complaint: a library that is present + # but unloadable -- wrong architecture, a missing dependency, + # no permission -- is otherwise indistinguishable from one + # that is absent, and is_supported() only logs what it caught. + raise CNDevError(CNDEV_ERROR_LIBRARY_NOT_FOUND) from load_error + finally: + # lock is always freed + libLoadLock.release() + + +## C function wrappers ## +def cndevInit(): + _LoadCndevLibrary() + + # Initialize the library + global _libInitialized, _libInitializedException + + # Checking the flag outside the lock lets two concurrent first callers both reach + # the driver, and the one that loses is answered with an already-initialized error + # that is then cached as the library's permanent state, disabling detection until + # the process restarts. The operator's binding uses sync.Once for the same reason, + # so the whole sequence is held here. + with libLoadLock: + if _libInitialized: + if _libInitializedException is not None: + # Re-raise a fresh copy: re-raising the same cached exception object + # appends a traceback frame on every call, and those frames retain the + # caller's locals, leaking memory over time. See gpustack/gpustack#5342. + from ..__utils__ import clone_exception + + raise clone_exception(_libInitializedException) from None + return + + try: + fn = _cndevGetFunctionPointer("cndevInit") + # The header names the parameter "reserved" and the operator passes 0. + # Version negotiation happens per struct, not here. + ret = fn(c_int32(0)) + _cndevCheckReturn(ret) + except Exception as e: + _libInitializedException = e + raise + finally: + _libInitialized = True + + +def cndevRelease(): + global _libInitialized, _libInitializedException + + with libLoadLock: + if not _libInitialized: + return + + # Initialization that never reached the driver left nothing to release. + if _libInitializedException is None: + # Unlike the sibling bindings' shutdown, this reaches the driver: + # cndevRelease frees what cndevInit allocated, as the operator's binding + # does. A driver too old to export it has nothing to free. + try: + fn = _cndevGetFunctionPointer("cndevRelease") + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + else: + # Clear the flags only once the driver has let go. Clearing them + # first would report the library uninitialized while it still + # holds what cndevInit allocated, and the next cndevInit would + # then initialize on top of it. + _cndevCheckReturn(fn()) + + _libInitialized = False + _libInitializedException = None + + +def cndevGetDeviceCount(): + c_card_info = c_cndevCardInfo_t() + c_card_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetDeviceCount") + ret = fn(byref(c_card_info)) + _cndevCheckReturn(ret) + return c_card_info.number + + +def cndevGetDeviceHandleByIndex(index): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByIndex") + ret = fn(index, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +@convertStrBytes +def cndevGetDeviceHandleByUUID(uuid): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByUUID") + ret = fn(uuid, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +@convertStrBytes +def cndevGetDeviceHandleByPciBusId(pciBusId): + c_handle = cndevDevice_t() + fn = _cndevGetFunctionPointer("cndevGetDeviceHandleByPciBusId") + ret = fn(pciBusId, byref(c_handle)) + _cndevCheckReturn(ret) + return c_handle.value + + +def cndevGetUUID(device): + c_uuid_info = c_cndevUUID_t() + c_uuid_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetUUID") + ret = fn(byref(c_uuid_info), device) + _cndevCheckReturn(ret) + # An empty field is not an identity: prefixing it would yield a bare "MLU-" that + # reads as a valid id and is the same for every card, so the usage join would write + # one card's metrics onto all of them. The caller's contract is that a card whose + # identity cannot be read is skipped, so say so rather than manufacture one. + if not c_uuid_info.uuid: + raise CNDevError(CNDEV_ERROR_NOT_FOUND) + # The driver reports the bare UUID. Every Cambricon tool, and the operator's + # UUID.String(), names the device with the "MLU-" prefix, so the prefix belongs here + # rather than in each caller. + return "MLU-" + c_uuid_info.uuid + + +def cndevGetPCIeInfoV2(device): + c_pcie_info = c_cndevPCIeInfoV2_t() + fn = _cndevGetFunctionPointer("cndevGetPCIeInfoV2") + ret = fn(byref(c_pcie_info), device) + _cndevCheckReturn(ret) + return c_pcie_info + + +def cndevGetPCIeBusId(device): + c_pcie_info = cndevGetPCIeInfoV2(device) + + domain = c_pcie_info.domain + bus = c_pcie_info.bus + dev = c_pcie_info.device + function = c_pcie_info.function + return f"{domain:04x}:{bus:02x}:{dev:02x}.{function:d}" + + +def cndevGetCardName(device): + c_card_name = c_cndevCardName_t() + c_card_name.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetCardName") + ret = fn(byref(c_card_name), device) + _cndevCheckReturn(ret) + return c_card_name + + +@convertStrBytes +def cndevGetCardNameString(cardName): + fn = _cndevGetFunctionPointer("cndevGetCardNameString") + fn.restype = c_char_p + return fn(cardName) + + +@convertStrBytes +def cndevGetCardNameStringByDevId(device): + fn = _cndevGetFunctionPointer("cndevGetCardNameStringByDevId") + fn.restype = c_char_p + return fn(device) + + +def cndevGetCardNameByDevId(device): + """ + Resolve the device's marketing name, whatever the driver offers. + + Not a CNDev entry point: it applies the precedence of the operator's + Device.GetCardName -- the name string by device, then the name enum resolved by the + library, then the enum resolved locally -- so the fallback lives in one place + instead of in every caller. + + Args: + device: + The device handle. + + Returns: + The device name, "MLU" when the driver reports an unknown name enum. + + Raises: + CNDevError: If the driver fails for a reason other than a missing symbol. + + """ + name = None + try: + name = cndevGetCardNameStringByDevId(device) + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + if name: + return name + + c_card_name = cndevGetCardName(device) + + try: + name = cndevGetCardNameString(c_card_name.id) + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + if name: + return name + + return _cndevCardNames.get(c_card_name.id, "MLU") + + +def cndevGetMemoryUsageV2(device): + c_memory_info = c_cndevMemoryInfoV2_t() + fn = _cndevGetFunctionPointer("cndevGetMemoryUsageV2") + ret = fn(byref(c_memory_info), device) + _cndevCheckReturn(ret) + return c_memory_info + + +def cndevGetCardHealthState(device): + c_health_state = c_cndevCardHealthState_t() + c_health_state.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetCardHealthState") + ret = fn(byref(c_health_state), device) + _cndevCheckReturn(ret) + return c_health_state + + +def cndevGetCardHealthStateV2(device): + c_health_state = c_cndevCardHealthStateV2_t() + try: + fn = _cndevGetFunctionPointer("cndevGetCardHealthStateV2") + except CNDevError as e: + if e.value != CNDEV_ERROR_FUNCTION_NOT_FOUND: + raise + # Mirror the operator's CardHealthStateHandler.V1: an older driver reports the + # same three fields through the V1 struct, so map them into the V2 shape and + # leave the incident report empty rather than failing the whole detection. + c_health_state_v1 = cndevGetCardHealthState(device) + c_health_state.health = c_health_state_v1.health + c_health_state.deviceState = c_health_state_v1.deviceState + c_health_state.driverState = c_health_state_v1.driverState + # V1 carries no overall verdict, and leaving the field at zero would read + # as CNDEV_HEALTH_RESULT_PASS: a false pass on every old driver, for any + # caller that reaches for it instead of the health bit. Derive it. + c_health_state.overallHealth = ( + CNDEV_HEALTH_RESULT_PASS + if c_health_state_v1.health == CNDEV_HEALTH_STATE_DEVICE_GOOD + else CNDEV_HEALTH_RESULT_FAIL + ) + return c_health_state + ret = fn(byref(c_health_state), device) + _cndevCheckReturn(ret) + return c_health_state + + +def cndevGetVersionInfo(device): + c_version_info = c_cndevVersionInfo_t() + c_version_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetVersionInfo") + ret = fn(byref(c_version_info), device) + _cndevCheckReturn(ret) + return c_version_info + + +def cndevGetDeviceUtilizationInfo(device): + c_util_info = c_cndevUtilizationInfo_t() + c_util_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetDeviceUtilizationInfo") + ret = fn(byref(c_util_info), device) + _cndevCheckReturn(ret) + return c_util_info + + +def cndevGetTemperatureInfo(device): + c_temperature_info = c_cndevTemperatureInfo_t() + # cndev.h declares this struct's version as IN, so it is stamped like every other + # versioned struct. The operator's GetTemperatureInfo is the one wrapper that leaves + # it zero, which its own GetVersionInfo comment suggests is an oversight surviving on + # its driver: an unstamped struct is answered with ERROR_UNSUPPORTED_API_VERSION. + # Temperature is a usage field the detector reads under suppression, so that error + # would read as no temperature rather than fail loudly. Confirmed on real hardware at + # checkpoint C2. + c_temperature_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetTemperatureInfo") + ret = fn(byref(c_temperature_info), device) + _cndevCheckReturn(ret) + return c_temperature_info + + +def cndevGetDevicePowerInfo(device): + c_power_info = c_cndevDevicePowerInfo_t() + fn = _cndevGetFunctionPointer("cndevGetDevicePowerInfo") + ret = fn(byref(c_power_info), device) + _cndevCheckReturn(ret) + return c_power_info + + +def cndevGetNUMANodeIdByDevId(device): + c_numa_node_id = c_cndevNUMANodeId_t() + c_numa_node_id.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetNUMANodeIdByDevId") + ret = fn(byref(c_numa_node_id), device) + _cndevCheckReturn(ret) + return c_numa_node_id + + +def cndevGetECCInfo(device): + c_ecc_info = c_cndevECCInfo_t() + c_ecc_info.version = CNDEV_VERSION_6 + fn = _cndevGetFunctionPointer("cndevGetECCInfo") + ret = fn(byref(c_ecc_info), device) + _cndevCheckReturn(ret) + return c_ecc_info diff --git a/gpustack_runtime/detector/pydcmi/__init__.py b/gpustack_runtime/detector/pydcmi/__init__.py index 9e54eb1..7c35e68 100644 --- a/gpustack_runtime/detector/pydcmi/__init__.py +++ b/gpustack_runtime/detector/pydcmi/__init__.py @@ -389,6 +389,21 @@ class c_dcmi_chip_info_v2(_PrintableStructure): ] +class c_dcmi_pcie_info(_PrintableStructure): + # The V1 struct, which -- unlike c_dcmi_pcie_info_all -- carries no domain, + # and orders the ids differently. Both are as dcmi_interface_api.h declares + # them. + _fields_: ClassVar = [ + ("deviceid", c_uint), + ("venderid", c_uint), + ("subvenderid", c_uint), + ("subdeviceid", c_uint), + ("bdf_deviceid", c_uint), + ("bdf_busid", c_uint), + ("bdf_funcid", c_uint), + ] + + class c_dcmi_pcie_info_all(_PrintableStructure): _fields_: ClassVar = [ ("venderid", c_uint), @@ -433,6 +448,16 @@ class c_dcmi_hbm_info(_PrintableStructure): ] +class c_dcmi_memory_info(_PrintableStructure): + # The V2 struct, which carries no available memory, only the utilization + # percentage, unlike c_dcmi_get_memory_info_stru of V3. + _fields_: ClassVar = [ + ("memory_size", c_ulonglong), + ("freq", c_uint), + ("utiliza", c_uint), + ] + + class c_dcmi_get_memory_info_stru(_PrintableStructure): _fields_: ClassVar = [ ("memory_size", c_ulonglong), @@ -847,6 +872,14 @@ def dcmi_get_device_type(card_id, device_id): return c_device_type.value +def dcmi_get_device_pcie_info(card_id, device_id): + c_pcie_info = c_dcmi_pcie_info() + fn = _dcmiGetFunctionPointer("dcmi_get_device_pcie_info") + ret = fn(card_id, device_id, byref(c_pcie_info)) + _dcmiCheckReturn(ret) + return c_pcie_info + + def dcmi_get_device_pcie_info_v2(card_id, device_id): c_pcie_info = c_dcmi_pcie_info_all() fn = _dcmiGetFunctionPointer("dcmi_get_device_pcie_info_v2") @@ -951,6 +984,14 @@ def dcmi_get_device_hbm_info(card_id, device_id): return c_hbm_info +def dcmi_get_device_memory_info_v2(card_id, device_id): + c_memory_info = c_dcmi_memory_info() + fn = _dcmiGetFunctionPointer("dcmi_get_device_memory_info_v2") + ret = fn(card_id, device_id, byref(c_memory_info)) + _dcmiCheckReturn(ret) + return c_memory_info + + def dcmi_get_device_memory_info_v3(card_id, device_id): c_memory_info = c_dcmi_get_memory_info_stru() fn = _dcmiGetFunctionPointer("dcmi_get_device_memory_info_v3") @@ -1119,6 +1160,17 @@ def dcmi_get_npu_work_mode(card_id): return c_work_mode.value +def dcmi_get_device_die(card_id, device_id): + # The V1 call takes no die type: it reports the SoC die only, which is what + # dcmi_get_device_die_v2 returns for DCMI_DIE_TYPE_VDIE. The struct is + # dcmi_soc_die_stru, laid out exactly as dcmi_die_id. + c_die_id = c_dcmi_die_id() + fn = _dcmiGetFunctionPointer("dcmi_get_device_die") + ret = fn(card_id, device_id, byref(c_die_id)) + _dcmiCheckReturn(ret) + return " ".join([hex(i)[2:] for i in c_die_id.soc_die]) + + def dcmi_get_device_die_v2(card_id, device_id, input_type): c_die_id = c_dcmi_die_id() fn = _dcmiGetFunctionPointer("dcmi_get_device_die_v2") diff --git a/gpustack_runtime/detector/thead.py b/gpustack_runtime/detector/thead.py index 73b4832..483b100 100644 --- a/gpustack_runtime/detector/thead.py +++ b/gpustack_runtime/detector/thead.py @@ -19,6 +19,7 @@ Topology, TopologyDistanceEnum, index_mig_devices, + merge_devices_usage, ) from .__utils__ import ( PCIDevice, @@ -28,7 +29,6 @@ get_numa_node_by_bdf, get_numa_nodeset_size, get_pci_devices, - get_physical_function_by_bdf, get_utilization, map_numa_node_to_cpu_affinity, ) @@ -81,9 +81,9 @@ def detect_pci_devices() -> dict[str, PCIDevice]: def __init__(self): super().__init__(ManufacturerEnum.THEAD) - def detect(self) -> Devices | None: + def detect_info(self) -> Devices | None: """ - Detect T-Head GPUs using pyhgml. + Detect T-Head GPUs' inventory using pyhgml, without usage metrics. Returns: A list of detected T-Head GPU devices, @@ -148,21 +148,12 @@ def detect(self) -> Devices | None: ) dev_numa = bitmask_to_str(list(dev_node_affinity)) - dev_temp = None - with contextlib.suppress(pyhgml.HGMLError): - dev_temp = pyhgml.hgmlDeviceGetTemperature( - dev, - pyhgml.HGML_TEMPERATURE_GPU, - ) - + # The power limit is inventory; the power actually drawn is + # usage, and belongs to detect_usage. dev_power = None - dev_power_used = None with contextlib.suppress(pyhgml.HGMLError): dev_power = pyhgml.hgmlDeviceGetPowerManagementDefaultLimit(dev) dev_power = dev_power // 1000 # mW to W - dev_power_used = ( - pyhgml.hgmlDeviceGetPowerUsage(dev) // 1000 - ) # mW to W dev_mig_mode = pyhgml.HGML_DEVICE_MIG_DISABLE with contextlib.suppress(pyhgml.HGMLError): @@ -170,6 +161,18 @@ def detect(self) -> Devices | None: dev_index = dev_idx + # Device.index is the enumeration index, while the driver's + # minor number goes to the appendix. Unlike NVIDIA's and + # Iluvatar's, the T-Head device node is named after the card + # ordinal and not after this number: the operator records it + # purely to PROVE a node addresses the card it describes, so it + # is left absent when the driver cannot answer rather than + # substituted by the enumeration index -- a substituted value + # would make a wrong ordinal look proven. + dev_minor_number = None + with contextlib.suppress(pyhgml.HGMLError): + dev_minor_number = pyhgml.hgmlDeviceGetMinorNumber(dev) + # Report the physical card, whether or not MIG is enabled. # MIG instances are partitioned on demand by the operator's # device-manager; they are not separate allocatable devices @@ -184,48 +187,19 @@ def detect(self) -> Devices | None: with contextlib.suppress(pyhgml.HGMLError): dev_cores = pyhgml.hgmlDeviceGetNumGpuCores(dev) - dev_cores_util = None - with contextlib.suppress(pyhgml.HGMLError): - dev_util_rates = pyhgml.hgmlDeviceGetUtilizationRates(dev) - dev_cores_util = dev_util_rates.gpu - if dev_cores_util is None: - debug_log_warning( - logger, - "Failed to get device %d cores utilization, setting to 0", - dev_index, - ) - dev_cores_util = 0 - - dev_mem = 0 - dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pyhgml.HGMLError): - dev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(dev) - dev_mem = byte_to_mebibyte( # byte to MiB - dev_mem_info.total, - ) - dev_mem_used = byte_to_mebibyte( # byte to MiB - dev_mem_info.used, - ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( - dev, - pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, - pyhgml.HGML_VOLATILE_ECC, - pyhgml.HGML_MEMORY_LOCATION_DRAM, - ) - if dev_mem_ecc_errors > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY - - dev_is_vgpu = False - if dev_bdf: - dev_is_vgpu = get_physical_function_by_bdf(dev_bdf) != dev_bdf + dev_mem, _ = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pyhgml.HGML_VOLATILE_ECC, + pyhgml.HGML_MEMORY_LOCATION_DRAM, + ) dev_appendix = { - "vgpu": dev_is_vgpu, "mig": dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE, "bdf": dev_bdf, } + if dev_minor_number is not None: + dev_appendix["minor_number"] = dev_minor_number if dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE: dev_mig_slots = 0 with contextlib.suppress(pyhgml.HGMLError): @@ -238,9 +212,7 @@ def detect(self) -> Devices | None: sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf, dev_numa, ) @@ -260,14 +232,9 @@ def detect(self) -> Devices | None: runtime_version_original=sys_runtime_ver_original, compute_capability=dev_cc, cores=dev_cores, - cores_utilization=dev_cores_util, memory=dev_mem, - memory_used=dev_mem_used, - memory_utilization=get_utilization(dev_mem_used, dev_mem), memory_status=dev_mem_status, - temperature=dev_temp, power=dev_power, - power_used=dev_power_used, appendix=dev_appendix, ), ) @@ -282,6 +249,115 @@ def detect(self) -> Devices | None: return ret + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch T-Head GPUs' usage using pyhgml, merged into the given devices. + + Args: + devices: + The devices to refresh, matched by UUID, GPU/compute instance + entries in ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, + or None if not supported. + + Raises: + If there is an error during fetching. + + """ + if not self.is_supported(): + return None + + if devices is None: + devices = self.detect_info() + if not devices: + return devices + + # The usage query enumerates the driver's devices on its own and returns + # them keyed by UUID, mirroring the operator's MonitorAccelerator: a + # metrics list is joined by device identity, never by index, as an index + # is not stable across a re-detection. + usages: Devices = [] + + try: + pyhgml.hgmlInit() + + dev_count = pyhgml.hgmlDeviceGetCount() + for dev_idx in range(dev_count): + dev = pyhgml.hgmlDeviceGetHandleByIndex(dev_idx) + + dev_uuid = pyhgml.hgmlDeviceGetUUID(dev) + + dev_cores_util = None + with contextlib.suppress(pyhgml.HGMLError): + dev_util_rates = pyhgml.hgmlDeviceGetUtilizationRates(dev) + dev_cores_util = dev_util_rates.gpu + if dev_cores_util is None: + debug_log_warning( + logger, + "Failed to get device %d cores utilization, setting to 0", + dev_idx, + ) + dev_cores_util = 0 + + dev_mem, dev_mem_used = _get_memory_info(dev) + dev_mem_status = _get_memory_status( + dev, + pyhgml.HGML_VOLATILE_ECC, + pyhgml.HGML_MEMORY_LOCATION_DRAM, + ) + + dev_temp = None + with contextlib.suppress(pyhgml.HGMLError): + dev_temp = pyhgml.hgmlDeviceGetTemperature( + dev, + pyhgml.HGML_TEMPERATURE_GPU, + ) + + dev_power_used = None + with contextlib.suppress(pyhgml.HGMLError): + dev_power_used = ( + pyhgml.hgmlDeviceGetPowerUsage(dev) // 1000 + ) # mW to W + + usages.append( + Device( + uuid=dev_uuid, + cores_utilization=dev_cores_util, + memory_used=dev_mem_used, + memory_utilization=get_utilization(dev_mem_used, dev_mem), + memory_status=dev_mem_status, + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + + dev_mig_mode = pyhgml.HGML_DEVICE_MIG_DISABLE + with contextlib.suppress(pyhgml.HGMLError): + dev_mig_mode, _ = pyhgml.hgmlDeviceGetMigMode(dev) + if dev_mig_mode != pyhgml.HGML_DEVICE_MIG_DISABLE: + dev_mig_slots = 0 + with contextlib.suppress(pyhgml.HGMLError): + dev_mig_slots = pyhgml.hgmlDeviceGetMaxMigDeviceCount(dev) + usages.extend( + _get_mig_usages( + dev, + dev_mig_slots, + dev_temp, + dev_power_used, + ), + ) + except pyhgml.HGMLError: + debug_log_exception(logger, "Failed to fetch devices usage") + raise + except Exception: + debug_log_exception(logger, "Failed to process devices usage fetching") + raise + + return merge_devices_usage(devices, usages) + def get_topology(self, devices: Devices | None = None) -> Topology | None: """ Get the Topology object between NVIDIA GPUs. @@ -296,7 +372,7 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: """ if devices is None: - devices = self.detect() + devices = self.detect_info() if devices is None: return None @@ -380,6 +456,71 @@ def get_topology(self, devices: Devices | None = None) -> Topology | None: return ret +def _get_memory_info( + dev: pyhgml.c_hgmlDevice_t, +) -> tuple[int, int]: + """ + Get a device's total and used memory. + + Args: + dev: + The HGML device handle. + + Returns: + The total and used memory in MiB, both 0 if unreadable. + + """ + with contextlib.suppress(pyhgml.HGMLError): + dev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(dev) + return ( + byte_to_mebibyte(dev_mem_info.total), + byte_to_mebibyte(dev_mem_info.used), + ) + + return 0, 0 + + +def _get_memory_status( + dev: pyhgml.c_hgmlDevice_t, + ecc_counter_type: int, + memory_location: int, +) -> DeviceMemoryStatusEnum: + """ + Get a device's memory health from its uncorrected ECC error counter. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. The usage query + cannot skip it: merging usage overwrites the status, so a status it did not + read would erase the one the information query found. + + Args: + dev: + The HGML device handle. + ecc_counter_type: + The ECC counter type to read, volatile or aggregate. + memory_location: + The memory location to read the counter of. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + with contextlib.suppress(pyhgml.HGMLError): + dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( + dev, + pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, + ecc_counter_type, + memory_location, + ) + if dev_mem_ecc_errors > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_gpm_metrics( metrics: list[int], dev: pyhgml.c_hgmlDevice_t, @@ -559,53 +700,46 @@ def _get_mig_devices( sys_runtime_ver, sys_runtime_ver_original, dev_cc, - dev_temp, dev_power, - dev_power_used, dev_bdf: str, dev_numa, ) -> list[dict]: """ - Enumerate the card's current MIG devices with the same detail a plain - device carries (profile name, uuid, compute/memory utilization, memory - health, temperature and power), returned as appendix entries of the - physical card rather than standalone devices. Empty when MIG is enabled - but no GPU instances exist yet. + Enumerate the card's current MIG devices with the same inventory detail a + plain device carries (profile name, uuid, cores, total memory and memory + health), returned as appendix entries of the physical card rather than + standalone devices. Empty when MIG is enabled but no GPU instances exist + yet. The operator has no T-Head equivalent of this enumeration; keeping it + is a deliberate divergence. + + An entry keeps a Device's shape, so the fields the usage query owns are + present at a Device's defaults: `_get_mig_usages` fills them. Each entry's `index` is the driver slot the MIG device was found at: index_mig_devices turns it into the device index once every card is detected. """ ret: list[dict] = [] - with contextlib.suppress(pyhgml.HGMLError): - for mdev_idx in range(dev_mig_slots): - mdev = None - with contextlib.suppress(pyhgml.HGMLError): - mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing a read + # used to abort the loop, so every later instance vanished from the + # inventory. An empty slot raises here as well, which is how it is + # skipped. + with contextlib.suppress(pyhgml.HGMLError): + mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) if not mdev: continue mdev_uuid = pyhgml.hgmlDeviceGetUUID(mdev) - mdev_mem = 0 - mdev_mem_used = 0 - mdev_mem_status = DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pyhgml.HGMLError): - mdev_mem_info = pyhgml.hgmlDeviceGetMemoryInfo(mdev) - mdev_mem = byte_to_mebibyte(mdev_mem_info.total) - mdev_mem_used = byte_to_mebibyte(mdev_mem_info.used) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - mdev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( - mdev, - pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, - pyhgml.HGML_AGGREGATE_ECC, - pyhgml.HGML_MEMORY_LOCATION_SRAM, - ) - if mdev_mem_ecc_errors > 0: - mdev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + mdev_mem, _ = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pyhgml.HGML_AGGREGATE_ECC, + pyhgml.HGML_MEMORY_LOCATION_SRAM, + ) mdev_appendix = { - "vgpu": True, "sliced": True, "mig": True, "bdf": dev_bdf, @@ -618,8 +752,6 @@ def _get_mig_devices( mdev_ci_id = pyhgml.hgmlDeviceGetComputeInstanceId(mdev) mdev_appendix["compute_instance_id"] = mdev_ci_id - mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) - mdev_name = "" mdev_cores = None mdev_gi = pyhgml.hgmlDeviceGetGpuInstanceById(dev, mdev_gi_id) @@ -693,20 +825,84 @@ def _get_mig_devices( "runtime_version_original": sys_runtime_ver_original, "compute_capability": dev_cc, "cores": mdev_cores, - "cores_utilization": mdev_cores_util, + "cores_utilization": 0, "memory": mdev_mem, - "memory_used": mdev_mem_used, - "memory_utilization": get_utilization(mdev_mem_used, mdev_mem), + "memory_used": 0, + "memory_utilization": 0, "memory_status": mdev_mem_status, - "temperature": dev_temp, + "temperature": None, "power": dev_power, - "power_used": dev_power_used, + "power_used": None, "appendix": mdev_appendix, }, ) return ret +def _get_mig_usages( + dev, + dev_mig_slots: int, + dev_temp, + dev_power_used, +) -> Devices: + """ + Fetch the usage of the card's current MIG devices, one UUID-keyed entry per + instance, to merge into the card's `appendix["mig_devices"]`. + + Args: + dev: + The HGML device handle of the card hosting them. + dev_mig_slots: + The number of MIG devices the card can host. + dev_temp: + The card's temperature. + dev_power_used: + The card's used power. + + Returns: + The MIG devices' usage, keyed by UUID. + + """ + ret: Devices = [] + for mdev_idx in range(dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing its UUID + # or its GPU instance id used to abort the loop, so every later instance + # kept the inventory's defaults -- 0 % and 0 MiB, reported idle while it + # may be running a workload. An empty slot raises here as well, which is + # how it is skipped. + with contextlib.suppress(pyhgml.HGMLError): + mdev = pyhgml.hgmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx) + if not mdev: + continue + + mdev_uuid = pyhgml.hgmlDeviceGetUUID(mdev) + + mdev_mem, mdev_mem_used = _get_memory_info(mdev) + mdev_mem_status = _get_memory_status( + mdev, + pyhgml.HGML_AGGREGATE_ECC, + pyhgml.HGML_MEMORY_LOCATION_SRAM, + ) + + mdev_gi_id = pyhgml.hgmlDeviceGetGpuInstanceId(mdev) + mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id) + + ret.append( + Device( + uuid=mdev_uuid, + cores_utilization=mdev_cores_util, + memory_used=mdev_mem_used, + memory_utilization=get_utilization(mdev_mem_used, mdev_mem), + memory_status=mdev_mem_status, + # A MIG device reports neither temperature nor power, so it + # carries the card's. + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + return ret + + def _get_gpu_instance_slice(dev_gi_prf_id: int) -> int: """ Get the number of slices for a given GPU Instance Profile ID. diff --git a/gpustack_runtime/envs.py b/gpustack_runtime/envs.py index 17782bc..3cfa266 100644 --- a/gpustack_runtime/envs.py +++ b/gpustack_runtime/envs.py @@ -55,10 +55,6 @@ e.g `{"cuda": "nvidia.com/devices", "rocm": "amd.com/devices"}`. Used to map the gpustack-runner's backend name to the corresponding resource key. """ - GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY: bool = True - """ - Use physical index priority at detecting devices. - """ ## Deployer GPUSTACK_RUNTIME_DEPLOY: str | None = None """ @@ -411,12 +407,6 @@ "hggc=alibabacloud.com/devices;", ), ), - "GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY": lambda: to_bool( - getenv( - "GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY", - "1", - ), - ), ## Deployer "GPUSTACK_RUNTIME_DEPLOY": lambda: getenv( "GPUSTACK_RUNTIME_DEPLOY", diff --git a/ruff.toml b/ruff.toml index a181dd6..80ced51 100644 --- a/ruff.toml +++ b/ruff.toml @@ -123,6 +123,7 @@ parametrize-names-type = "csv" "gpustack_runtime/deployer/__patches__.py" = ["ALL"] "gpustack_runtime/detector/pyamdsmi/*.py" = ["ALL"] "gpustack_runtime/detector/pyamdgpu/*.py" = ["ALL"] +"gpustack_runtime/detector/pycndev/*.py" = ["ALL"] "gpustack_runtime/detector/pydcmi/*.py" = ["ALL"] "gpustack_runtime/detector/pyhgml/*.py" = ["ALL"] "gpustack_runtime/detector/pyhsa/*.py" = ["ALL"] diff --git a/specs/2026-08-14-detector-alignment-and-workload-exit-status.md b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md new file mode 100644 index 0000000..027bec4 --- /dev/null +++ b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md @@ -0,0 +1,1088 @@ +# Spec: Align runtime detector with the operator's device manager, and surface workload exit status + +Status: Built +Type: Feature + +## Summary + +`gpustack_runtime.detector` and `gpustack-operator/pkg/devicemanager/detector` describe the same +hardware through two independently-grown code paths, and the runtime side has drifted: it misses +capability queries the operator performs, it classifies cards as virtual/vGPU, it fetches +utilization metrics it cannot avoid paying for, and it indexes devices by driver minor number — +a scheme that conflicts with `CUDA_DEVICE_ORDER=PCI_BUS_ID` +([gpustack/gpustack#6041](https://github.com/gpustack/gpustack/issues/6041)). Separately, a +`Workload` never reports why a container died, so a Kubernetes deployment whose image cannot be +pulled stays silent in the UI ([gpustack/gpustack#5869](https://github.com/gpustack/gpustack/issues/5869)). + +This change realigns the detect path with the operator vendor by vendor, drops the vGPU notion in +favour of whole-card reporting (MIG instances stay in the card's `appendix`), splits detection into +an information query and a usage query so callers can skip metrics, retires the physical-index +environment switch and pins `CUDA_DEVICE_ORDER=PCI_BUS_ID` on the NVIDIA workloads that can see +every card of the host, and exposes structured per-container exit status — including Kubernetes +image-pull failures backed by Pod Events. + +## Motivation + +### Goals + +- **Audited detect-path parity.** For every one of the nine supported manufacturers, the runtime's + detect path performs the same driver queries, version fallbacks and eligibility filters as the + operator's, or records a deliberate, documented divergence. Target users: GPUStack worker + operators who expect `gpustack-runtime detect` and the operator's `Devices` object to describe the + same card identically. +- **Whole-card inventory.** Detection reports the card the driver enumerates, with no + virtual/vGPU/PF/VF classification. MIG instances of a MIG-enabled card keep their current + representation inside `Device.appendix["mig_devices"]`. +- **Metrics are opt-out.** A caller that only needs inventory (name, uuid, total memory, cores, + versions, topology keys) can obtain it without paying for utilization, temperature or power + queries — which on NVIDIA alone cost a 100 ms GPM sampling window per card. +- **Two composable queries.** Detection is split into an information query and a usage query; + the default keeps today's behaviour by running both and merging the result. +- **Predictable device ordering.** `Device.index` is the detector's enumeration index. The + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` switch is removed; driver-physical numbering + that device-node paths need moves into the appendix. A container that can see every card of the + host — because it is privileged, or because it requested all devices — gets its manufacturer's + ordering pinned by default (`CUDA_DEVICE_ORDER=PCI_BUS_ID` on NVIDIA), so the runtime, the vendor + tooling and the workload all number the cards the same way. +- **Diagnosable workloads.** `WorkloadStatus` carries per-container exit information on Docker, + Podman and Kubernetes; a Kubernetes workload whose image cannot be pulled reports `Failed` with + the ErrImagePull/ImagePullBackOff reason and the corresponding Pod Event message. + +Success is testable: `make test` green, with new tests covering the split API, the removal of the +vGPU classification, the appendix physical indexes, the env propagation, and the exit-status / +image-pull parsing (fixture-driven, no hardware required). + +### Non-Goals + +- Aligning `get_topology` / topology data between the two code bases. Explicitly out of scope. +- Changing how the operator partitions MIG or how it reports devices; the operator is the reference, + not the subject. +- Introducing a metrics/monitor loop in the runtime equivalent to the operator's + `Detector.Start`. The runtime exposes the queries; scheduling stays with the caller. +- Supporting Ascend vNPU (`/dev/vdavinci`) detection. It is removed with the vGPU logic. +- Changing `Device`'s existing field names or the JSON shape of already-reported fields, beyond the + additions and removals listed below. + +## Proposal + +The runtime's detector becomes a faithful, worker-side twin of the operator's device manager for +everything on the detect path, with a query split that lets callers choose how much they pay for. +The deployer stops relying on driver minor numbers for ordering, and starts reporting why a +container is not running. + +### User Stories + +#### Story 1 +As a GPUStack worker operator, I want `gpustack-runtime detect` to report the same card name, memory +size and health as the operator's device manager on the same node, so that a discrepancy between +the two is a real hardware event and not a code-path difference. + +#### Story 2 +As a GPUStack worker operator running MetaX cards on a virtualization-enabled host, I want the +detector to report my physical cards, so that the worker does not come up with zero devices. + +#### Story 3 +As a caller of the runtime library that only needs an inventory (for example to build CDI specs), I +want to ask for device information without utilization metrics, so that a detect pass does not spend +a GPM sampling window per card. + +#### Story 4 +As a caller that renders a live device dashboard, I want to refresh only the usage fields of devices +I already know, so that I can poll cheaply. + +#### Story 5 +As a GPUStack user deploying a model onto all of a node's NVIDIA cards, I want CUDA inside the +inference container to number the GPUs the way `nvidia-smi` and the runtime's detection do, so that +"GPU 2" means the same card everywhere and I do not have to configure anything to get it. + +#### Story 6 +As a GPUStack user deploying a model on a Kubernetes cluster with an unpullable backend image, I +want the deployment to report the pull failure and its reason, so that I understand why the model +never becomes ready instead of watching it hang. + +#### Story 7 +As a GPUStack user whose inference container crashed, I want the workload status to tell me the exit +code and the termination reason (`OOMKilled`, `Error`, signal), so that I can act without shelling +into the node. + +### Core Features & Acceptance Criteria + +#### F1 — Detect-path parity audit and fixes + +Compared method-by-method against the operator (`DetectAccelerator` only; `MonitorAccelerator` is +covered by F3). Known gaps, each to be closed or documented as a deliberate divergence: + +| Vendor | Gap in the runtime today | Expected | +| --- | --- | --- | +| NVIDIA | No GDDR ECC capacity restore. Operator reads `GetMemoryBusWidth` + `GetEccMode` and restores `memory * 16 / 15` when the bus is `< 1024`-bit and ECC is on. | **Withdrawn during PR review, and the divergence kept deliberately.** The restore was implemented in `T5`, then removed: the operator's corrected figure is a *display* value that takes no part in allocation, while `memory` here does, and `memory - memory_used` has to mean free space. Adding back capacity that is not reachable would over-commit every GDDR card with ECC on. `memory` is therefore what the driver reports, i.e. what the card can allocate. | +| NVIDIA | `nvmlDeviceGetMemoryInfo` called without a version preference; operator prefers V2 with a V1 fallback. | Prefer the v2 memory structure, fall back to v1, matching the operator's `GetMemoryInfoV`. The argument is the **packed struct-version constant** `pynvml.nvmlMemory_v2`, not the literal `2` — `pynvml` assigns it straight into the struct's `version` field, which the driver validates, so a literal is rejected. Probe for the constant rather than raising the dependency floor. | +| NVIDIA | `nvmlDeviceGetPciInfo` called without a version preference. | **No change, recorded as a documented divergence.** `pynvml` exposes no v2 accessor at all: `nvmlDeviceGetPciInfo` is an alias of `nvmlDeviceGetPciInfo_v3`, and `nvmlPciInfo_v2_t` is declared but unused. v3 is a superset of the operator's preferred v2, and the operator does not read the struct's string anyway — its `GetBusId()` formats the BDF from domain/bus/device, so the struct version is invisible to what either side consumes. Reaching v2 would mean hand-rolling the struct inside the detector, against this repo's "`pynvml` is a thin wrapper over the upstream package" constraint. | +| NVIDIA | Falls back to *host* memory (`get_memory()`) when the device total reads 0; the operator skips such a device. | Keep, and record it as a deliberate divergence with the reason in a code comment (WSL/iGPU tolerance). | +| Ascend | No device-type filter; the operator skips a device whose `GetType() != NPU_TYPE`. | Skip non-NPU devices, matching the operator's exact semantics: skip **only** when the type call succeeds and the type differs; a device whose type is unreadable is kept. | +| Ascend | Two divergences found while building `T6`, both deliberately left unfixed and carried to the **Deferred** table below. **(a)** The operator's `MonitorAccelerator` reads utilization through a v2→v1 fallback — `dcmi_get_device_utilization_rate_v2` fills a `MultiUtilizationInfo` in one call, else four separate `dcmi_get_device_utilization_rate` calls. `pydcmi` binds no `_v2` utilization at all and the runtime reads only the AICORE rate, so it is permanently on the operator's V1 path. **(b)** `_get_device_memory_status` tries HBM then DDR ECC unconditionally, where the operator picks the ECC device type from whichever memory query succeeded. | Both recorded, neither implemented: **(a)** is outside the four fallbacks `F1` names for Ascend and needs a `pydcmi` addition of its own; **(b)** is pre-existing behaviour and out of `T6`'s scope. | +| Ascend | `*_v2`/`*_v3` DCMI calls with no V1 fallback (vdie, PCIe, chip info, memory); the operator falls back. | Add the V1 fallbacks so older drivers still detect. **Three of the four need entry points `pydcmi` does not bind yet** — `dcmi_get_device_die`, `dcmi_get_device_pcie_info` and `dcmi_get_device_memory_info_v2`, plus the `c_dcmi_pcie_info` / `c_dcmi_memory_info` structs — so `T6` owns `pydcmi/__init__.py` to add them raw, keeping the fallback *policy* in `ascend.py` where the fake can prove it. On the memory fallback, neither struct carries `memory_available`: report `used = memory_size * utiliza / 100` from the utilization percent rather than mirroring the operator, whose `Memory_available == 0` makes `memory_size - available` report **every card as 100 % used** on that path. Verified against `npu-smi` at `C2`. | +| MetaX | **`continue`s on `MXSML_VIRTUALIZATION_MODE_PF`** — i.e. it drops the physical function, the whole card — where the operator drops `VF`. | No virtualization-mode filter at all (see F2); the physical cards are reported. | +| MThreads | Skips `MTML_VIRT_ROLE_HOST_VIRTDEVICE` when `mpcCap != MPC_TYPE_INSTANCE`; the operator skips `GUEST_VIRTDEVICE`. **Worse than a wrong filter: `mpcCap` is not a field of `c_mtmlDeviceProperty_t`** — the real names are `virtCapability, virtRole, mpcCapability, mpcType, rsvd`, and `_PrintableStructure` defines no `__getattr__`, so the read raises `AttributeError`, which `except Exception: raise` propagates and **the whole MThreads detect pass fails**. A non-virtualized card short-circuits on the `and`, which is why it has never been hit. Also note the constant compared against, `MTML_MPC_TYPE_INSTANCE`, belongs to `mpcType`, not to a capability field. | No virtRole filter at all (see F2), which removes the crash and the drop together. **Separately, and deliberately not fixed by `T9`:** `detect_info` frees the device handle in its `finally` (`mthreads.py:135`) and then uses it to open the memory context (`:142`) — a use-after-free on the MTML handle that predates this work. `T9` leaves it untouched because fixing it is a behaviour change outside its ask, and its new `detect_usage` does not reproduce it (it holds the handle open across both contexts and frees it once). **Needs its own task**, which can also carry `mtmlDeviceGetPowerUsage`'s unconverted raw value. | +| Iluvatar | Memory read without the V2→V1 fallback the operator performs. | Add the fallback. | +| Hygon / AMD | Device name resolved from HSA/ASIC only; the operator prefers a `pci.ids` lookup (`GetPCIDeviceNames`) first, then HSA `ProductName`, then the amdgpu marketing name. | The operator's full precedence, **including the libdrm step**: `pci.ids` → HSA `ProductName` → `amdgpu_get_marketing_name` → ASIC market name (AMD) / `rsmi_dev_name_get` (Hygon). The libdrm call was originally left unbound and recorded as a divergence; PR review asked for it, so `pyamdgpu` now binds it and both detectors reach for it. Verified against libdrm on the AMD host, which answers `AMD Radeon RX 7800 XT` for both cards. | +| Cambricon | Not a binding at all: a `cnmon info -e -m -u -j` shell-out parsing `cnmon_info.json`, with a `TODO(thxCode)` where the sample output should be. No driver version, no Neuware version, no cores, no power, no BDF, no NUMA, no health/ECC, no PCIe bus id. | New hand-written `pycndev` ctypes binding (following `pydcmi`/`pymxsml`) and a rewritten `cambricon.py` calling `GetDeviceCount`, `GetDeviceHandleByIndex`, `GetUUID`, `GetPCIeInfoV`, `GetCardName`, `GetMemoryInfoV`, `GetCardHealthStateV`, `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, `GetPowerInfo`, plus the Neuware version from `/usr/local/neuware/version.txt`. `cndev.h` in the operator's `binding/cndev` is the header of record. Four decisions taken while building `T3`, all recorded rather than assumed: **cores and the power limit stay unreported** — `pycndev` binds no core-count entry point and the operator reads neither for this vendor, so closing those two needs a binding addition of its own; **`driver_version` carries `major.minor.build`** where the operator formats only `major.minor`, since it is free precision from the same query and every other vendor here reports the driver's full string; **health comes from `cndevGetCardHealthStateV2().health` alone, with no ECC read**, because that is literally what the operator does (`memoryUnhealthy = healthInfo.Health == 0`); and **memory needs no unit conversion** — the header says MB but the operator, `cnmon` and this repo's Ascend detector all treat the value as MiB, so converting would under-report ~4.8 % against the operator on the same host, which is the very discrepancy Story 1 exists to remove. `C2` confirms the memory figure against `cnmon`. | +| Cambricon | Per-device failure policy diverges from every sibling **on purpose**: `T3` follows the operator and `continue`s past a card whose required reads fail, where the other eight detectors let the error propagate. | A single faulty card therefore costs one device on Cambricon and **all** devices on the other eight. Graceful degradation is very likely the right behaviour everywhere and the other eight should move toward it, but that is a repo-wide behaviour change deserving its own task — not something to slip in per vendor. | +| THead | Enumerates MIG-style GPU/compute instances, which the operator does not. | Keep — this is the appendix mechanism F2 preserves. | +| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. | + +Acceptance criteria: +- ~~A written parity table lives in the repo (module docstring or `docs/`) listing, per vendor, the + driver calls the detect path makes and any deliberate divergence from the operator, with its + reason.~~ **Withdrawn after the build, by request:** `T11` wrote `docs/detector-parity.md` and it + was published, then dropped as not belonging in the shipped documentation. Nothing it recorded is + lost — the deferred items it carried are listed under **Deferred** below, and every divergence it + explained is stated in a comment at the code that makes it. +- Every gap above is either fixed or annotated as deliberate; no gap is left silently open. +- Sample/fixture-driven tests assert that NVIDIA memory is reported as the driver reports it, the Ascend non-NPU skip, and + the MetaX card no longer being dropped. + +#### F2 — Whole-card reporting, no vGPU classification + +- `Device.appendix` no longer carries a `vgpu` key, for any vendor. +- All virtual-card detection is removed: NVIDIA's `_is_vgpu` PCI-capability sniff, the Ascend vNPU + (`dcmi` vdev query) branch, the SR-IOV physical-function comparisons in AMD / Hygon / Iluvatar / + THead, MetaX's virtualization-mode check, MThreads' `virtRole` check. +- MIG instances keep the existing mechanism unchanged: the physical card is the reported device, its + instances live in `appendix["mig_devices"]`, numbered by `index_mig_devices`, and + `expand_mig_devices` still substitutes them for callers that address instances. +- `deployer/cdi/ascend.py` always uses `/dev/davinci{index}`; the `/dev/vdavinci` branch is removed. +- No detector filters devices by virtualization mode or role; whatever the driver enumerates is + reported. + +#### F3 — Split information and usage queries + +- `Detector` grows two methods and keeps `detect` as the composing entry point: + - `detect_info() -> Devices | None` — abstract; identity, capability and inventory fields only. + - `detect_usage(devices: Devices | None = None) -> Devices | None` — utilization, used memory, + memory utilization, temperature, used power, plus a refreshed `memory_status`. With `devices` + given it merges into them (matched by `uuid`, MIG entries included); with `None` it runs + `detect_info()` first. + - `detect(usage: bool = True) -> Devices | None` — `detect_info()`, then `detect_usage()` merged in + when `usage` is true. Default behaviour is byte-for-byte what `detect()` returns today. +- Field split: **information** = `manufacturer, index, name, uuid, driver_version, runtime_version, + runtime_version_original, compute_capability, cores, memory, power, appendix`; **usage** = + `cores_utilization, memory_used, memory_utilization, temperature, power_used`, and + `memory_status` is produced by both (mirroring the operator, which reports `Unhealthy` from both + `DetectAccelerator` and `MonitorAccelerator`). +- Module level: `detect_devices(fast=True, manufacturer=None, usage=True)`; the new keyword threads + through to every detector. +- `gpustack-runtime detect` grows `--no-usage`; the table format renders `N/A` for the omitted + columns instead of `0`. +- With `usage=False`, no utilization/temperature/power driver call is made — asserted per vendor by + monkeypatched-binding tests that fail if such a call happens. + +#### F4 — Retire the physical-index switch, propagate `CUDA_DEVICE_ORDER` + +- `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` is removed from `envs.py` and from all three + call sites (NVIDIA, Ascend, Iluvatar). `Device.index` is the detector's enumeration index. +- Driver-physical numbering that a device-node path or a vendor tool needs is exposed in the + appendix instead — mirroring the operator, which keeps a sequential `Index` next to + `PhysicalIndexes`. AMD's `card_id`/`renderd_id` and Ascend's `card_id`/`device_id` already do this; + NVIDIA, Iluvatar and THead gain their minor number the same way — but **only NVIDIA's and + Iluvatar's name a device node.** THead's `/dev/alixpu_ppu{N}` is named after the card ordinal, i.e. + the enumeration index; the operator's own comment at its `GetMinorNumber` call site says so, and it + keeps the number purely to *prove* a node addresses the card it describes, by comparing it against + the node's character-device minor. So THead records `minor_number` in the appendix and its CDI path + keeps reading `Device.index`. +- Every consumer of `dev.index` that means "device node number" (`deployer/cdi/*`) is reviewed + against the appendix values, so no `/dev/...` path changes meaning. Two of them actually read the + retired numbering, because the switch defaults to **on** and therefore describes today's real + behaviour: `cdi/ascend.py`'s `/dev/davinci{...}` (physical id from the DCMI logic id) and + `cdi/iluvatar.py`'s `/dev/iluvatar{...}` (NVML minor number). Both move to the appendix key in the + same change that retires the switch, so no host ever sees a window where a workload is handed + another card's device node. `cdi/amd.py`, `cdi/hygon.py` and `cdi/metax.py` already build their + paths from `appendix["card_id"]` / `["renderd_id"]`, `cdi/thead.py` was never on the switch, and + NVIDIA has no CDI generator in this repository. +- A container that ends up seeing **every** accelerator of the host gets its manufacturer's device + ordering pinned by default, on Docker, Podman and Kubernetes alike. "Every accelerator" is exactly + the condition the deployers already resolve per device request: the container is `privileged`, or + its device request is `all`. On NVIDIA that means `CUDA_DEVICE_ORDER=PCI_BUS_ID`; no other + manufacturer documents an ordering switch, so no other variable is injected. + - It is a **default**, not an override: a container that declares `CUDA_DEVICE_ORDER` itself keeps + its own value. + - A container that was given a specific subset is left alone — it does not see the cards it was + not given, and the deployer already scopes its visible-devices env. The criterion is the subset, + not who allocated it: a device plugin handing out *every* card of the node (an `all` request + under the KDP injection policy) still leaves the container seeing everything, so the ordering is + pinned there too. + - The worker's own environment is irrelevant to this decision. The ordering is a property of what + the deployer hands the container, so the deployer decides it. + - The operator pins the same variable, by the same never-overwrite rule, at a **complementary** + point: its NVIDIA device plugin sets it on *sliced* containers, because HAMi-core fills its + `CUDA_DEVICE_MEMORY_LIMIT_` table in NVML enumeration order but reads a limit back by CUDA + ordinal, and the two coincide only under `PCI_BUS_ID`. So on an operator-managed cluster a sliced + workload gets it from the plugin and a privileged / all-devices workload gets it from here. Same + name, same value, both set-if-absent — the two injections cannot disagree. +- `Device.index`'s docstring loses the physical-index paragraph. +- **A skipped device does not renumber its survivors.** The operator compacts its `Index`, so its + second card becomes `Index 0` when the first is skipped. The runtime keeps the driver's enumeration + index, because compacting would make the index *unstable across passes*: a transiently faulty card + would shift every later card's index down and back again on recovery, and `DevicesMaterial`'s + `runtime_values` / `backend_values` / `numa_affinities` are all keyed by `str(dev.index)`. A + non-contiguous index is a static fact a caller can hold; a shifting one silently repoints a cached + index at a different card. Decided while building `T3`, the only detector that skips a device. + +#### F5 — Workload exit status and Kubernetes image-pull failures + +- `WorkloadStatus` gains a per-container exit list — one entry per container that has terminated or + is blocked from starting — carrying: container name, operation token, exit code, reason, message, + started/finished timestamps, and restart count. Absent containers are simply not listed. +- Docker and Podman populate it from each container's `State` (`ExitCode`, `Error`, `OOMKilled`, + `StartedAt`, `FinishedAt`, `RestartCount`), including init containers. +- Kubernetes populates it from `container_statuses` / `init_container_statuses`, reading + `last_state.terminated` and `state.terminated` for exit codes and `state.waiting` for + reason/message. +- A Kubernetes workload whose image cannot be pulled (`ErrImageNeverPull`, `ErrImagePull`, + `ImagePullBackOff`, `InvalidImageName`, `RegistryUnavailable`) reports `state = Failed` — not + `Pending` — with the reason and message in `state_message`, and the matching Pod `Event` message + appended. `ErrImageNeverPull` was added by the end-of-build review: it is the one genuinely + unrecoverable reason of the five (`imagePullPolicy: Never` with the image absent), and it was + missing while recoverable reasons were present. +- The appended Event is selected by `involvedObject.uid` as well as name, chosen by timestamp rather + than list order — the API guarantees none — and prefers the kubelet's `Failed` Event, so a stale + `FailedScheduling` cannot stand in as the diagnosis. +- Pod Events are read only when the Pod is in such a blocked state, and an Events call denied by + RBAC degrades gracefully (state and reason still reported, Event detail omitted, one debug log). +- `deploy/manifests/kubernetes.yaml` grants `get`/`list` on core `events`. +- Fixture-driven tests cover: Docker exited-nonzero, Docker OOMKilled, K8s terminated exit code, + K8s `ImagePullBackOff` → `Failed` with reason, and an Events call raising `403`. + +### Notes / Constraints / Caveats + +- Python ≥ the project's floor; `uv` for dependency management; `ruff` + `pre-commit` for lint. +- Vendor bindings are hand-written `ctypes` modules under `gpustack_runtime/detector/py*` (see + `pydcmi`, 1.2k lines; `pymxsml`, 1.6k lines) except `pynvml`/`pymtml`, which are thin wrappers over + upstream PyPI packages. `pycndev` follows the hand-written pattern. +- The operator is the reference implementation and is not modified by this work. +- All new fields are `dataclass_json` dataclass fields, defaulted so existing serialized payloads + keep deserializing. +- Hardware-dependent tests stay `pytest.mark.skipif(not .is_supported())`, as today; new + behaviour is covered by fixtures and monkeypatched bindings so CI without GPUs still exercises it. + +### Boundaries + +- **Always:** keep MIG instances in `appendix["mig_devices"]` with the current numbering; keep + `detect()`'s default output identical to today's; keep the topology **data** untouched — a + `get_topology` body may be repointed at `detect_info()` so topology stops paying for metrics it + never reads, but the `Topology` it returns must not change; keep `Device`'s existing field names. +- **Ask first:** before changing what a `/dev/...` device-node path resolves to for any vendor; + before adding an RBAC verb beyond core `events`; before touching the operator repository. +- **Never:** align `get_topology`'s data or semantics with the operator's, or change what it returns; + add a monitor/scheduling loop to the runtime; override a + `CUDA_DEVICE_ORDER` the container already declares; inject an ordering variable into a container + that was given a device subset. + +### Risks and Mitigations + +- Ascend `/dev/davinci{N}` numbering may follow the driver's physical id, not the logic id → keep + the physical id in the appendix and have the CDI generator read it from there; confirm against a + real 910B/910C node before merging. +- Retiring the physical-index switch shifts the *keys* other consumers derive from `Device.index`: + `DevicesMaterial.runtime_values` / `backend_values` / `numa_affinities` are keyed by + `str(dev.index)` (`deployer/__types__.py:1414-1427`), and every CDI generator emits its devices as + `ConfigDevice(name=str(dev.index))`. On NVIDIA and Iluvatar those keys go from the minor number to + the enumeration index, and on Ascend from the physical id to the logic id → harmless within one + process, because a single detect pass feeds both sides, and it is the direction issue #6041 asks + for; but a caller that persisted indexes across the upgrade sees them move, so it belongs in the + release note next to the ordering change. +- Removing the Ascend vNPU branch drops detection on hosts that pre-split NPUs → documented as a + Non-Goal and a release note; partitioning is the operator's device manager's job. +- Removing MetaX's `PF` skip and MThreads' `virtRole` skip changes reported device counts on + virtualization-enabled hosts → sample-driven tests pin the expected inventory for both. +- A new `pycndev` binding cannot be validated without Cambricon hardware → derive it from the + operator's `cndev.h`, unit-test struct layouts and parsing against captured samples, and keep the + hardware path behind `is_supported()`. +- The `usage=False` path could silently keep issuing metric calls → tests assert on the binding + functions actually invoked, not just on the returned values. +- Reading Pod Events adds API traffic per status poll → only for blocked Pods, only field-selected + to the Pod, and skipped entirely once the workload runs. +- Pinning `CUDA_DEVICE_ORDER=PCI_BUS_ID` changes the CUDA ordinals inside a container that previously + ran under the `FASTEST_FIRST` default, so a workload addressing cards by CUDA ordinal sees a + different card for the same number on a heterogeneous host → that is the defect being fixed (the + ordinals now agree with `nvidia-smi` and with the runtime's own detection), but it is a visible + behaviour change on such hosts, so it belongs in the release note; a container that needs the old + behaviour declares `CUDA_DEVICE_ORDER=FASTEST_FIRST` itself and keeps it. +- `T1` mechanically renames `detect` → `detect_info` in all nine vendor files, so it conflicts with any + in-flight branch touching a detector → land it first and alone, before the per-vendor tasks start. +- `samples/` is documentation-only — no test reads it, and it has already drifted from the code + (`detect_output_nvidia_h100_mig.json` still uses the retired standalone-MIG shape with + `gpu_instance_index` / `compute_instance_index`) → `T12` realigns every sample and adds a + schema-guard test, so a future drift fails the suite instead of rotting silently. +- `pytest.ini` lists an `integration_tests` testpath that does not exist → create the directory (with + the integration scenarios below) or drop the entry, before adding integration tests. +- No coverage measurement exists today (`addopts` carries no `--cov`), so the Test Plan's per-package + numbers are targets, not deltas → introduce `pytest-cov` as an optional prerequisite, or accept + hand-review of the new tests' reach. +- Real-hardware validation cannot run during the build (macOS dev machine, no vendor drivers) → every + task verifies through fixtures/monkeypatched bindings, and `C2` gates the merge on the user-provided + environments. + +## Design Details + +### Commands + +**Environment.** Implementation and the whole automated suite run **locally** (macOS dev machine, `uv`). +No vendor driver exists there, so every detector's hardware path is skipped by `is_supported()` and all +new behaviour must be provable through fixtures and monkeypatched bindings — that constraint shapes the +Test Plan. Real-hardware validation is checkpoint `C2`, on the environments below. + +**`C2` environments** (supplied 2026-08-15). The checkout lives at `~/github.com/gpustack/runtime` on +each target and its git tree must be brought in line with the branch under test. + +| Vendor | Access | +| --- | --- | +| THead (ppu) | `root@192.168.100.151`, jumping through `frank@192.168.50.17` | +| AMD | `frank@192.168.50.17` | +| NVIDIA | `frank@192.168.50.13` and `frank@192.168.50.16` | +| Ascend | `root@120.241.57.28` | + +Two ways to get the code running there, per target: + +- **From source**, where the target has Python: `make install`, then run the CLI from the checkout's + `.venv`. +- **From a container image**, built on `frank@192.168.50.17`: + `PACKAGE_NAMESPACE=thxcode PACKAGE_TAG=dev- make package`, pushing `thxcode/runtime:dev-` + to Docker Hub, then running that image on the target. + +```bash +# On each target, per pass +gpustack-runtime detect --format json +gpustack-runtime detect --no-usage --format json +gpustack-runtime topology --format json +# Health only means anything with the ECC read switched on -- see F1's all-vendors row +GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false gpustack-runtime detect --format json +``` + +**One question only the AMD/Hygon host can answer.** Both build their UUID as +`f"GPU-{rsmi_dev_unique_id_get(idx)[2:]}"`, and that wrapper returns `hex(value)`, so a driver +answering `0` yields `"GPU-0"` for *every* card — a truthy but non-unique identity, which is the key +the usage merge joins on. The merge now drops an ambiguous id rather than broadcasting one card's +metrics onto the rest, so the harm is contained either way, but whether the ids are actually distinct +is a measurement, not a guess: compare the `uuid` values `detect --format json` reports on the AMD +host. If they collide, the identity needs a distinct fallback (the BDF is read on the same pass) and +that becomes its own task. + +**These four cover four of the nine vendors**, plus Hygon by proxy (below). Iluvatar, MetaX, MThreads +and Cambricon have no environment, so their detect paths stay proven only by fixtures — and in +particular the MetaX temperature Open Question and Cambricon's "the driver says MB but means MiB" +reading **cannot be settled by `C2` as scoped**. Both must keep their markers until hardware appears. + +**Hygon's logic is exercised on the AMD host, by proxy.** A Hygon DCU *is* a ROCm device, and +`hygon.py` runs on the same binding stack AMD does — `pyrocmsmi` + `pyhsa`, with `pyamdgpu` for the +cores fallback. The only vendor-specific gate is one PCI vendor id: `hygon.py`'s +`get_pci_devices(vendor="0x1d94")` against `amd.py`'s `"0x1002"`. Temporarily pointing Hygon's at +`0x1002` on the AMD host makes `HygonDetector.is_supported()` pass and runs every line of its +`detect_info` / `detect_usage` against a real driver. + +- **What that proves:** the Hygon path executes without raising on real hardware; the `pyrocmsmi` / + `pyhsa` field names it reads exist; its fallback chains resolve; the usage merge and the + `memory_status` recomputation work; and its CDI generator emits paths. +- **What it does not prove:** any Hygon-specific *value*. The names, ECC counters, memory and core + counts will be the AMD card's, so nothing about a K100AI's own reporting is settled by it. +- The edit is a throwaway on the target, recorded in the evidence and reverted; it must never be + committed. **The trick does not generalise** — Iluvatar, MetaX, MThreads and Cambricon each load + their own vendor library, which is absent on these hosts, so there is nothing for a vendor-id swap + to reach. + +```bash +make prepare # REQUIRED FIRST: writes the gitignored gpustack_runtime/_version_appendix.py. + # Without it any `uv run` fails at the editable build with + # "Forced include not found: .../gpustack_runtime/_version_appendix.py". +make deps # uv sync --all-packages && uv lock && uv tree +make lint # uv run pre-commit run --all-files --show-diff-on-failure + # (ruff-check --fix --unsafe-fixes, ruff-format, codespell — it rewrites files) +make test # uv run pytest (pytest.ini: pythonpath=., addopts=--no-header -vvv) +make docs # mkdocs build + +# Task-scoped verification +uv run pytest tests/gpustack_runtime/detector -q +uv run pytest tests/gpustack_runtime/deployer -q +uv run pytest tests/gpustack_runtime/detector/test_samples.py -q + +# Manual smoke on the dev machine (no devices -> empty list, must not traceback) +uv run gpustack-runtime detect --format json +uv run gpustack-runtime detect --no-usage --format json +``` + +### Project Structure + +``` +gpustack_runtime/ +├── detector/ +│ ├── __init__.py # detect_devices/detect_backend/expand_mig_devices facade +│ ├── __types__.py # Device, Devices, Topology, Detector ABC, index_mig_devices +│ ├── __utils__.py # PCI/NUMA/version/unit helpers +│ ├── .py # amd, ascend, cambricon, hygon, iluvatar, metax, mthreads, nvidia, thead +│ └── py/ # bindings. Hand-written ctypes: pyamdgpu, pyamdsmi, pydcmi, pyhgml, +│ # pyhsa, pyixml, pymxsml, pyrocmsmi (+ NEW pycndev). +│ # Thin wrappers over upstream PyPI: pynvml, pymtml. +├── deployer/ +│ ├── __types__.py # Container/Workload{Plan,Status}, Deployer ABC, DevicesMaterial +│ ├── docker.py # DockerWorkloadStatus.parse_state +│ ├── podman.py +│ ├── kuberentes.py # KubernetesWorkloadStatus.parse_state, Pod/Event reads +│ └── cdi/.py # CDI spec generation, consumes Device.index/appendix +├── cmds/detector.py # `detect` / `topology` sub-commands +└── envs.py # GPUSTACK_RUNTIME_* env surface +deploy/manifests/kubernetes.yaml # RBAC for the deployer +tests/gpustack_runtime/ +├── detector/ +│ ├── fixtures/__init__.py # load() helper only — no data files yet +│ ├── samples/ # detect_output_*.json / topology_output_*.json captured from real +│ │ # hardware. Documentation-only today: NO test reads them, so they +│ │ # have drifted (detect_output_nvidia_h100_mig.json still uses the +│ │ # retired standalone-MIG shape). T12 realigns them and adds a guard. +│ └── test_.py # hardware-gated smoke tests (skipif not is_supported()) +└── deployer/ + ├── fixtures/ + └── test_workload_status.py, test_privileged.py, test_runtime_class.py, test_utils.py +``` + +### Code Style + +The query split, as the ABC composes it (`detector/__types__.py`): + +```python +class Detector(ABC): + @abstractmethod + def detect_info(self) -> Devices | None: + """ + Detect devices' inventory, without usage metrics. + """ + raise NotImplementedError + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + """ + Fetch the usage of the given devices, merged into them in place. + + Args: + devices: + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. + If None, detects the devices' information first. + + Returns: + The devices carrying usage, or None if not supported. + + """ + return devices + + def detect(self, usage: bool = True) -> Devices | None: + devices = self.detect_info() + if usage and devices: + self.detect_usage(devices) + return devices +``` + +A vendor's information query, showing the operator-mirroring comment convention: + +```python +class NVIDIADetector(Detector): + def detect_info(self) -> Devices | None: + """ + Detect NVIDIA GPUs' inventory using pynvml, without usage metrics. + + Returns: + A list of detected NVIDIA GPU devices, + or None if not supported. + + Raises: + If there is an error during detection. + + """ + if not self.is_supported(): + return None + + ret: Devices = [] + try: + pynvml.nvmlInit() + ... + dev_mem = 0 + with contextlib.suppress(pynvml.NVMLError): + dev_mem_info = pynvml.nvmlDeviceGetMemoryInfo( + dev, + version=pynvml.nvmlMemory_v2, # packed struct version, not a literal 2 + ) + # What the driver reports, i.e. what the card can allocate. The + # operator restores the ~1/16 that ECC parity carves out of a GDDR + # part, but that figure is display-only there, while this one takes + # part in allocation -- see F1's first row. + dev_mem = byte_to_mebibyte(dev_mem_info.total) + except pynvml.NVMLError: + debug_log_exception(logger, "Failed to fetch devices") + raise + return ret +``` + +Conventions: `from __future__ import annotations as __future_annotations__` first; `@dataclass_json +@dataclass` models with per-field docstrings; `contextlib.suppress(Error)` around optional +driver calls; `debug_log_exception` / `debug_log_warning` for diagnostics; Google-style docstrings +with `Args:` / `Returns:` / `Raises:`; comments explain *why*, referencing the operator when the +behaviour is a deliberate mirror. + +### Implementation Plan + +Two independent tracks — the detector track (T*) and the deployer track (D*) share no file and start in +parallel — joined by two checkpoints. The detector track is sequenced **expand → migrate → contract**, +because splitting `detect` is one mechanical change whose blast radius covers all nine vendor files: no +vertical slice can land green while half the vendors still override `detect()` and the other half +implement `detect_info()`. + +#### Detector track + +- [x] **T1 · Prefactor: split the Detector query surface (expand), retire the physical-index switch** + Blocked by: None + Owns: `gpustack_runtime/detector/__types__.py`, `gpustack_runtime/detector/__init__.py`, + `gpustack_runtime/detector/__utils__.py`, `gpustack_runtime/detector/*.py`, + `gpustack_runtime/envs.py`, `gpustack_runtime/deployer/cdi/ascend.py`, + `gpustack_runtime/deployer/cdi/iluvatar.py`, + `tests/gpustack_runtime/detector/test_detector_types.py`, + `tests/gpustack_runtime/detector/test_mig_devices.py` + Gate: review + Scope: add `detect_info()` / `detect_usage(devices=None)` / `detect(usage=True)` to the `Detector` + ABC as shown in Code Style; rename every vendor's `detect()` to `detect_info()` with no body + change, so `detect()` output is unchanged and `usage=False` is an accepted no-op until each + vendor migrates; thread `usage=` through `detect_devices()` and call it with `usage=False` from + `get_devices_topologies()`, and point the nine vendors' own `get_topology()` at `detect_info()` + so topology stops paying for metrics once T3–T10 land (identical body during expand, so no + behaviour change); add a module-level `merge_devices_usage(devices, usages)` to `__types__.py` + joining by `uuid` across cards **and** `appendix["mig_devices"]` entries, mirroring the + operator's `MonitorAccelerator`, which returns a separate UUID-keyed metrics list that consumers + join by identity, never by index — T3–T10 call it from their `detect_usage`; delete + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` from `envs.py` (both the `TYPE_CHECKING` stub + and the `variables` registry entry that actually resolves it) and its three call sites (NVIDIA, + Ascend, Iluvatar), moving the driver-physical number into the appendix (`minor_number` for + NVIDIA/Iluvatar — THead's belongs to T10, whose index the switch never touched; Ascend adds + `physical_id` beside its existing `card_id`/`device_id`); **keep the two `/dev/...` paths that + read the retired numbering byte-for-byte** — `cdi/ascend.py`'s `/dev/davinci{...}` and + `cdi/iluvatar.py`'s `/dev/iluvatar{...}` become one-line reads of the appendix key, since the + switch defaults to **on** and those paths are physical numbering today (T6/T7 then only re-pick + which key); add a `get_pci_device_name()` pci.ids lookup helper to `__utils__.py` for T4; drop + the physical-index paragraph from `Device.index`'s docstring, wording it as the index the + detector enumerates the device at, without claiming contiguity (Ascend reports the DCMI logic + id, and MIG numbering punches holes). + Acceptance: `detect()`'s payload is unchanged **by construction** — no vendor overrides + `detect()`, so it is `detect_info()`'s payload plus a no-op merge; asserted structurally (every + vendor implements `detect_info`, none overrides `detect`, the composition preserves the info + fields) rather than against nine golden fixtures, which would need the `fixtures/bindings.py` + fakes the Test Plan lists as a prerequisite and T3–T10 build. "Unchanged" means every + pre-existing field and appendix key keeps its value, the appendix *gains* the physical-number + key, and an index moves only where F4 intends it to. Plus: `detect(usage=False)` is accepted by + every detector; the env var appears nowhere in the tree; `Device.index` is the enumeration index + and the physical number is reachable from the appendix; `/dev/davinci{N}` and `/dev/iluvatar{N}` + resolve to the same nodes as before the change; `get_pci_device_name()` resolves a known + vendor:device pair, prefers a subsystem name over the device name, and returns `""` on a miss. + Verify: `make prepare && uv run pytest tests/gpustack_runtime/detector -q` + +- [x] **T2 · PoC: `pycndev` ctypes binding** + Blocked by: None + Owns: `gpustack_runtime/detector/pycndev/**`, `tests/gpustack_runtime/detector/test_pycndev.py`, + `ruff.toml` + Gate: review + Scope: hand-written ctypes binding following `pydcmi` / `pymxsml`, derived from the operator's + `binding/cndev/cndev.h` (the header of record): library loader, init/release, `GetDeviceCount`, + handle-by-index, `GetUUID`, `GetPCIeInfoV` (V2), `GetCardName`, `GetMemoryInfoV` (V2), + `GetCardHealthStateV` (V2), `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, + `GetPowerInfo`. + Acceptance: the module imports cleanly with no Cambricon library present and makes no call at + import time; every struct's `ctypes.sizeof` and field offsets match `cndev.h`; a missing library + surfaces as the binding's own error type, not a bare `OSError`. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_pycndev.py -q` + +- [x] **T3 · Cambricon detector on `pycndev`** + Blocked by: T1, T2 + Owns: `gpustack_runtime/detector/cambricon.py`, + `tests/gpustack_runtime/detector/test_cambricon.py` + Gate: review + Scope: replace the `cnmon info -e -m -u -j` shell-out (and its `TODO(thxCode)` placeholder) with + `pycndev`. `detect_info`: uuid, card name, total memory, driver version, Neuware version from + `/usr/local/neuware/version.txt` (`\d+\.\d+\.\d+`, as the operator reads it), PCIe bus id, NUMA + node, health from `GetCardHealthStateV`. `detect_usage`: core utilization, used memory, memory + utilization, temperature, power. Drop `appendix["vgpu"]`. + Acceptance: with a monkeypatched `pycndev`, `detect_info()` returns a Device carrying name, uuid, + memory, driver_version, runtime_version, bdf, numa and a real `memory_status`, and issues no + utilization/temperature/power call; `detect()` adds the usage fields; `cnmon` is invoked nowhere + in the tree. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_cambricon.py -q` + +- [x] **T4 · AMD + Hygon: pci.ids name precedence, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/amd.py`, `gpustack_runtime/detector/hygon.py`, + `gpustack_runtime/deployer/cdi/amd.py`, `gpustack_runtime/deployer/cdi/hygon.py`, + `tests/gpustack_runtime/detector/test_amd.py`, `tests/gpustack_runtime/detector/test_hygon.py` + Scope: adopt the operator's name precedence — pci.ids (`get_pci_device_name` from T1) → HSA + product name → amdgpu marketing name → ASIC market name; delete `appendix["vgpu"]` and the + `get_physical_function_by_bdf` comparison; move `amdsmi_get_gpu_metrics_info`, used power, + `rsmi_dev_busy_percent_get`, `rsmi_dev_temp_metric_get` and used memory into `detect_usage`, + keeping the power *limit* in `detect_info`; confirm the CDI generators read + `appendix["card_id"]` / `["renderd_id"]`, not `Device.index`. + Acceptance: pci.ids hit wins over the HSA name; with pci.ids absent the previous name is + returned; no appendix carries `vgpu`; `detect_info()` makes no utilization/temperature/used-power + call; the CDI spec for both vendors is byte-identical to the pre-change output for the same + fixture. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_amd.py tests/gpustack_runtime/detector/test_hygon.py -q` + +- [x] **T5 · NVIDIA: memory version preference, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/nvidia.py`, `tests/gpustack_runtime/detector/test_nvidia.py`, + `tests/gpustack_runtime/detector/test_mig_devices.py` + Gate: review + Scope: prefer the v2 memory structure with a v1 fallback; delete `_is_vgpu` and + `appendix["vgpu"]`; keep the MIG appendix mechanism, moving the MIG entries' usage fields + (`cores_utilization`, `memory_used`, `memory_utilization`, `temperature`, `power_used`) into + `detect_usage`; keep the host-memory fallback for a zero total and record in a comment that it + is a deliberate divergence from the operator (WSL / iGPU tolerance); record + `appendix["minor_number"]`. + **The GDDR ECC capacity restore this task originally carried was reverted during PR review** + — see F1's first row — so `_restore_ecc_reserved_memory()` and its bus-width/ECC-mode + acceptance cases exist nowhere in the tree; `memory` is what the driver reports. + Acceptance: monkeypatched NVML — the v2 memory structure is preferred and a binding without it + falls back to v1; a GDDR card with ECC on reports the driver's total unrestored; `detect_info()` + issues no GPM/utilization/temperature/power-usage call; `vgpu` absent from every appendix; a + MIG-enabled card still carries `appendix["mig_devices"]` with its instances numbered by + `index_mig_devices`. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_nvidia.py tests/gpustack_runtime/detector/test_mig_devices.py -q` + +- [x] **T6 · Ascend: NPU type filter, V1 fallbacks, drop the vNPU branch, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/ascend.py`, `gpustack_runtime/detector/pydcmi/__init__.py`, + `gpustack_runtime/deployer/cdi/ascend.py`, + `tests/gpustack_runtime/detector/test_ascend.py` + Gate: review + Scope: skip a device whose DCMI type is not NPU, as the operator does; add V1 fallbacks for vdie, + PCIe, chip-info and memory so older drivers still detect; delete the vdev (vNPU) branch and + `appendix["vgpu"]`; make the CDI generator always emit `/dev/davinci{...}` from the appendix + value that matches the driver's device-node numbering — T1 already wired it to `physical_id` to + preserve today's paths, so this is a one-line re-pick if `C2` shows the logic id is the right + key (see Open Questions); move utilization, temperature and power into `detect_usage`. + Acceptance: a non-NPU device in the card is not reported; a driver exposing only the V1 calls + still yields a Device; no `/dev/vdavinci` path is produced anywhere; `detect_info()` makes no + utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_ascend.py -q` + +- [x] **T7 · Iluvatar: memory V2→V1 fallback, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/iluvatar.py`, `gpustack_runtime/deployer/cdi/iluvatar.py`, + `tests/gpustack_runtime/detector/test_iluvatar.py` + Scope: add the memory V2→V1 fallback the operator performs; delete `appendix["vgpu"]` and the + `get_physical_function_by_bdf` comparison; move utilization, temperature and used power into + `detect_usage`; confirm the CDI `/dev/iluvatar{N}` path still reads the appendix minor number + T1 wired it to. + Acceptance: v2 unavailable → v1 result returned; no appendix carries `vgpu`; `detect_info()` + makes no utilization/temperature/used-power call; the CDI spec is unchanged for the same fixture. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_iluvatar.py -q` + +- [x] **T8 · MetaX: remove the inverted PF skip, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/metax.py`, `gpustack_runtime/deployer/cdi/metax.py`, + `tests/gpustack_runtime/detector/test_metax.py` + Gate: review + Scope: delete the `continue` on `MXSML_VIRTUALIZATION_MODE_PF` — it currently drops the physical + function, i.e. the whole card, where the operator drops `VF` — and apply no virtualization-mode + filter at all; delete `appendix["vgpu"]`; move core utilization, used memory, temperature and + board power into `detect_usage`. + Acceptance: a fixture reporting `mode == PF` yields that card as a Device (it is dropped today); + a fixture reporting `mode == VF` also yields a Device; `detect_info()` makes no + utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_metax.py -q` + +- [x] **T9 · MThreads: remove the virtRole skip, de-vGPU, usage split** + Blocked by: T1 + Owns: `gpustack_runtime/detector/mthreads.py`, `tests/gpustack_runtime/detector/test_mthreads.py` + Gate: review + Scope: delete the `MTML_VIRT_ROLE_HOST_VIRTDEVICE` / `mpcCap` skip and `appendix["vgpu"]`, and + apply no virtRole filter; move GPU utilization, temperature, used memory and used power into + `detect_usage`. + Acceptance: a fixture reporting a host virt device with `mpcCap != MPC_TYPE_INSTANCE` is now + reported (it is dropped today); `detect_info()` makes no utilization/temperature/power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_mthreads.py -q` + +- [x] **T10 · THead: de-vGPU, usage split (MIG appendix preserved)** + Blocked by: T1 + Owns: `gpustack_runtime/detector/thead.py`, `gpustack_runtime/deployer/cdi/thead.py`, + `tests/gpustack_runtime/detector/test_thead.py` + Scope: delete `appendix["vgpu"]` (both the card's and the instance entries') and the + `get_physical_function_by_bdf` comparison; keep the GPU/compute-instance enumeration in the + appendix — the operator has no THead equivalent and this is the mechanism F2 preserves; move + utilization, temperature and used power into `detect_usage`, instance entries included; record + `appendix["minor_number"]` where the binding exposes it. Creates `test_thead.py`, which does not + exist today. + Acceptance: no appendix carries `vgpu`; instance entries survive with `sliced` intact (the + topology path keys off it); `detect_info()` makes no GPM/utilization/temperature/used-power call. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_thead.py -q` + +- [x] **T11 · Contract: make the split mandatory, `detect --no-usage`, ~~parity table~~** + Blocked by: T3, T4, T5, T6, T7, T8, T9, T10 + Owns: `gpustack_runtime/detector/__types__.py`, `gpustack_runtime/detector/__utils__.py`, + `gpustack_runtime/cmds/detector.py`, ~~`docs/**`, `mkdocs.yml`~~, + `tests/gpustack_runtime/detector/test_detector_cli.py`, + `tests/gpustack_runtime/detector/test_detector_types.py` + Gate: review + Scope: mark **both** `detect_info` and `detect_usage` `@abstractmethod`, so a future vendor + cannot silently skip the split — `raise NotImplementedError` would only fail when called, which + is exactly the silence being removed, and a vendor with genuinely no usage query is better off + declaring a two-line stub than inheriting a no-op that makes `detect()` look like it measured + something. All nine already implement both, so the cost is zero. Also migrate the two + expand-state fixtures in `test_detector_types.py` (`_UnimplementedDetector`, `_InfoOnlyDetector`) + that exist precisely to pin the un-contracted state; delete `get_physical_function_by_bdf` if no caller + remains; add `detect --no-usage`, rendering `N/A` instead of `0` in the table for the omitted + columns; ~~write the per-vendor parity table (driver calls made, and every deliberate divergence + with its reason) into `docs/`~~. + **The parity document was withdrawn after the build, by request** — `docs/detector-parity.md`, + its `mkdocs.yml` nav entry and the test asserting every vendor appeared in it were all removed, + so `docs/` and `mkdocs.yml` carry nothing from this work. Nothing it recorded is lost: the + deferred items it carried are the **Deferred** table below, and every divergence it explained + is stated in a comment at the code that makes it. + Acceptance: a `Detector` subclass without `detect_info` fails to instantiate; `detect --no-usage` + prints `N/A` for utilization/temperature and issues no metric call; ~~the parity table names all + nine vendors and every divergence recorded in F1~~ (withdrawn with the document); + **no detector can emit a `vgpu` key** — + each vendor task could only assert its own output, so this guard lands here, scoped to the + shipped package (`gpustack_runtime/**/*.py`). Deliberately **not** a literal tree-wide text + grep: `test_ascend.py` plants a stale `vgpu` in an appendix on purpose, to prove it cannot + resurrect `/dev/vdavinci`, and the captured `samples/` are covered by `T12`'s own schema guard. + Verify: `uv run pytest tests/gpustack_runtime/detector -q && make docs` + +- [x] **T12 · Align `samples/detect_output_*.json` with the new Device contract** + Blocked by: T3, T4, T5, T6, T7, T8, T9, T10 + Owns: `tests/gpustack_runtime/detector/samples/**`, + `tests/gpustack_runtime/detector/test_samples.py` + Scope: realign every `detect_output_*.json` field-by-field to the post-change shape — no `vgpu`; + MIG instances inside the card's `appendix["mig_devices"]` (`detect_output_nvidia_h100_mig.json` + still uses the retired standalone-MIG shape with `gpu_instance_index` / + `compute_instance_index`); the physical-index appendix keys present; NVIDIA totals left as the + driver reports them. Add the schema-guard test that loads every sample and deserializes it, + so the samples stop drifting silently. `topology_output_*.json` is untouched (topology is a + Non-Goal). + Acceptance: every `detect_output_*.json` deserializes into `Device`; none carries `vgpu`, or + `gpu_instance_index` / `compute_instance_index` at the top level; MIG samples carry + `appendix["mig_devices"]`; the guard test fails when a sample grows a key `Device` cannot hold. + Verify: `uv run pytest tests/gpustack_runtime/detector/test_samples.py -q` + +#### Deployer track + +- [x] **D1 · Pin the device ordering for containers that see every card** + Blocked by: None + Owns: `gpustack_runtime/deployer/__types__.py`, `gpustack_runtime/deployer/docker.py`, + `gpustack_runtime/deployer/podman.py`, `gpustack_runtime/deployer/kuberentes.py`, + `tests/gpustack_runtime/deployer/test_visible_devices_ordering.py` + Gate: review + Scope: add `Deployer.map_visible_devices_ordering(runtime_envs) -> dict[str, str]` beside the + existing `map_backend_visible_devices` / `map_visible_devices_affinities` — it resolves each + runtime visible-devices env name to its manufacturer through `get_manufacturer()` and returns + `{"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}` when NVIDIA is among them, `{}` otherwise. Call it from all + three deployers' device-request loops (`docker.py`, `podman.py`, `kuberentes.py`), right beside + the existing `if r_v != "all" and privileged:` backend-visible-devices block, under the + complementary condition `r_v == "all" or privileged` — the deployers' own resolution of "this + container sees every card". Never overwrite a value the container already declares (the + Kubernetes site appends to a `V1EnvVar` list, so it needs a name check). + Acceptance: NVIDIA + `all` → the container carries `CUDA_DEVICE_ORDER=PCI_BUS_ID` on all three + deployers; NVIDIA + privileged with a specific device request → injected; NVIDIA + a specific + device request without privilege → not injected; a non-NVIDIA manufacturer → nothing injected; + a container declaring `CUDA_DEVICE_ORDER=FASTEST_FIRST` keeps its own value; the auto-mapping + resource key on a mixed-manufacturer node injects once, not once per manufacturer. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_visible_devices_ordering.py -q` + +- [x] **D2 · `WorkloadStatus` exit-status model** + Blocked by: D1 + Owns: `gpustack_runtime/deployer/__types__.py`, + `tests/gpustack_runtime/deployer/test_workload_status.py` + Gate: review + Scope: add a `WorkloadStatusExit` `dataclass_json` dataclass (name, token, exit_code, reason, + message, started_at, finished_at, restart_count) and a defaulted `WorkloadStatus` field holding a + list of them. Serial after D1 purely to keep a single writer on `__types__.py`. + Acceptance: a status with no exits round-trips; a payload serialized before this change still + deserializes; the new field is excluded from no existing consumer's expectations. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_workload_status.py -q` + +- [x] **D3 · Docker: populate exit status** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/docker.py`, + `tests/gpustack_runtime/deployer/test_docker_status.py` + Scope: fill the exit list from each container's `State` (`ExitCode`, `Error`, `OOMKilled`, + `StartedAt`, `FinishedAt`, `RestartCount`) for init and run containers, and carry the reason into + `state_message`. `parse_state`'s existing verdicts do not change. + Acceptance: exited with 137 and `OOMKilled` → the existing state verdict is preserved and an exit + entry carries 137 plus the `OOMKilled` reason; a running container contributes no entry; a + container whose `State` lacks the keys degrades to an entry with the code alone. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_docker_status.py -q` + +- [x] **D4 · Podman: populate exit status** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/podman.py`, + `tests/gpustack_runtime/deployer/test_podman_status.py` + Scope: the D3 change applied to the Podman deployer's own status class. Runs concurrently with D3 + and D5 — disjoint files. + Acceptance: same as D3, against Podman's container attrs. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_podman_status.py -q` + +- [x] **D5 · Kubernetes: exit status, image-pull failure, Pod Events, RBAC** + Blocked by: D2 + Owns: `gpustack_runtime/deployer/kuberentes.py`, `deploy/manifests/kubernetes.yaml`, + `tests/gpustack_runtime/deployer/test_kubernetes_status.py` + Gate: review + Scope: fill the exit list from `container_statuses` / `init_container_statuses`, reading + `state.terminated` and `last_state.terminated` for codes and `state.waiting` for reason and + message; make `parse_state` return `Failed` — not `Pending` — for `ErrImagePull`, + `ImagePullBackOff`, `InvalidImageName` and `RegistryUnavailable`, with reason and message in + `state_message`; read Pod Events, field-selected to the Pod, only in that blocked state and + append the Event message; degrade a `403` to a debug log; grant `get`/`list` on core `events` in + the manifest. + Acceptance: Pending + `ImagePullBackOff` → `Failed` carrying the reason and the registry message; + `terminated(1, "Error")` → an exit entry with 1; Events raising `ApiException(403)` → state and + reason still reported and nothing raised; a Running Pod triggers no Events call. + Verify: `uv run pytest tests/gpustack_runtime/deployer/test_kubernetes_status.py -q` + +#### End-of-build review + +Three axes ran over the whole branch diff: the **spec** axis (`spec-reviewer`), the **standards** axis +(`agent-skills:code-reviewer`), and two external cross-checks (codex and kimi). The spec axis returned +Missing 0 / Unasked 0 / Wrong 1 — a self-contradiction of this document's own Boundaries on +`get_topology`, fixed at the source. The other three produced 20 findings between them, of which 14 +were real and are fixed above the C2 line, each with a regression test that fails without the fix: + +- **The Ascend `/dev/davinciN` fallback**, flagged High by codex and Critical by the standards axis + while kimi read it as preserved behaviour. All three were partly right: the pre-change default path + *did* build the node from the logic id, because the physical id was only used when + `GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY` was set. So it is a pre-existing defect this work + preserved and can now close, not a regression it introduced — and it is exactly what this document's + "**Ask first:** before changing what a `/dev/...` device-node path resolves to" exists to catch. +- Two identity defects in the new usage join: an ambiguous UUID broadcasting one card's metrics onto + every card, and `cndevGetUUID` manufacturing a bare `"MLU-"` that is such an id. +- Two availability defects: a usage-query failure discarding an inventory already in hand, and every + Cambricon card being skipped reading as "no hardware". +- Two reporting defects a consumer would have acted on: `--format json` ignoring `--no-usage`, and a + container that merely exited non-zero carrying no reason and so no `state_message`. +- **Two tests that could not fail**, which matter more than the findings they hid: the usage-join test + used identical payloads for every card, so a positional join, a reversed mapping and a + broadcast-to-all all satisfied it; and the memory-status merge test ran with health checks off, where + both queries answer `healthy` without a driver call. + +#### Deferred — each needs its own task + +These were found during the build or its review, judged real, and deliberately **not** fixed here, +because each is a behaviour change wider than this work's scope. They were recorded in a parity +document that has since been dropped, so the list lives here: + +| Vendor | Deferred item | +| --- | --- | +| All | Eight bindings check `_libInitialized` **outside** the load lock, so two concurrent first callers can both reach the driver's init and the loser's error is cached as permanent state. `pycndev` is fixed; `pyamdsmi`, `pydcmi`, `pyhgml`, `pyixml`, `pymtml`, `pymxsml`, `pynvml` and `pyrocmsmi` are not — they keep upstream `pynvml`'s shape on purpose so they can be re-synced. One mechanical sweep, provable per binding with the same two-caller test. | +| Ascend | Utilization is read through the operator's **V1** path only, and only the AICORE rate; `pydcmi` binds no `_v2` utilization entry point at all. Needs a binding addition. | +| Ascend | `_get_device_memory_status` tries HBM ECC then DDR ECC unconditionally, where the operator picks the ECC device type from whichever memory query succeeded. Pre-existing. | +| Cambricon | The only vendor that degrades **per card**; the other eight lose the whole pass on one faulty card. Graceful degradation is very likely right everywhere, but that is a repo-wide change. | +| Cambricon | A card skipped by the **usage** pass keeps the information query's zeroes rather than reading as unmeasured. Its `memory_status` still carries the health that query read, so a failing card is not fully masked — but telling "not measured" from 0 needs `cores_utilization`, `memory_used` and `memory_utilization` to become optional, which is a cross-vendor change to `Device`, its consumers and every sample. | +| Cambricon | `cores` and the power **limit** are not reported: `pycndev` binds no core-count entry point and the operator reads neither. | +| MetaX | **The temperature divisor is unresolved** — see Open Questions. Needs hardware. | +| MThreads | `detect_info` frees the MTML handle in its `finally` and then uses it to open the memory context: a use-after-free. Pre-existing, and `detect_usage` does not reproduce it. | +| MThreads | `mtmlDeviceGetPowerUsage` is reported without unit conversion. Same follow-up as above. | + +- [x] **C1 · Checkpoint: whole suite, lint, CLI smoke** + Blocked by: T11, T12, D3, D4, D5 + Owns: None + Gate: review + Acceptance: `make lint` leaves the tree clean, `make test` is green, and both + `gpustack-runtime detect --format json` and `--no-usage --format json` run on the dev machine + returning an empty list without a traceback. + Verify: `make prepare && make lint && make test` + +- [x] **C2 · Checkpoint: hardware validation on the user-provided environments** + Blocked by: C1 + Owns: `tests/gpustack_runtime/detector/samples/**` + Gate: review + Scope: on each environment the user provides, run `detect` with and without `--no-usage`, compare + the reported card name, memory, cores and health against the operator's `Devices` object on the + same node — **health only with `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false`**, since the + default skips the ECC read entirely and would report `healthy` unconditionally (see F1's + all-vendors row), deploy one workload with an unpullable image to confirm the reported failure, and + refresh any sample whose real output differs from T12's hand-alignment. Also settles the Ascend + `/dev/davinci{N}` Open Question. + **Three value-level refreshes `T12` deliberately refused to invent, so they are `C2`'s to + capture:** the keys absent from the realigned samples (`minor_number` on NVIDIA and THead, + `physical_id` on Ascend, and the card-level fields of the two rebuilt MIG cards in + `detect_output_nvidia_h100_mig.json`) — their absence is honest, not a discrepancy; the two + `1c.3g.40gb` MIG instances, still carrying the *compute-instance* profile name from the capture + while today's detector reports the *GPU-instance* one; and the AMD / Hygon `name` values, which + now prefer the host's local pci.ids board name and so cannot be derived from a sample at all. + Acceptance: per environment, name/memory/health match the operator's report for every card; + `--no-usage` omits the metric fields; the unpullable-image workload reports `Failed` with the + pull reason; samples committed from real output. + + **Result.** All five environments ran `detect`, `detect --no-usage`, `topology` and a + `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK=false` pass, every one exiting 0 with an empty stderr: + 2 × AMD RX 7800 XT, 1 × RTX 5090 D, 2 × RTX 4090 48 GB, 8 × Ascend 910B2, 16 × T-Head + PPU-ZW810E. `--no-usage` omitted exactly the five usage keys and kept `memory`, `memory_status` + and `power`. Health read the same verdict with the ECC check on as off on every card, so no + card on these hosts is carrying an uncorrected error. + + Four things it measured that no fixture could: + + 1. **T-Head's minor number is not its enumeration index**, on any of the 16 cards. They ran one + apart here (index 3 → minor 4, index 15 → minor 16), and the operator's allocator records the + same relation on its own measured host — but **that offset is an observation about a host and + a driver, not a rule**, and nothing on either side may compute one number from the other. The + operator says so at `allocator/thead/mig.go:640-645`, and its own note explains the offset on + that host: `/dev/alixpu` holds minor 0 of the same character-device major, so the per-card + nodes start one along. What the measurement establishes is only the load-bearing part — the + two numbers differ — which is why `/dev/alixpu_ppu` is named by the **card ordinal** and + the minor is read from `hgmlDeviceGetMinorNumber` as the card's *identity*. Using the + identity as the name lands on the neighbouring card, and on the last card of a 16-card host + names a `ppu16` that does not exist at all. + 2. **Ascend reports `physical_id` for all 8 NPUs**, so the invariant the Ascend fix relies on + holds on a real driver. Logic and physical id agree here (0–7), which is why the fix is also + provably non-regressive on this host. + 3. **AMD's per-card UUIDs are distinct** (`GPU-5c88007d760374f3` / `GPU-d99e7fe92c7bdf75`), so + `rsmi_dev_unique_id_get` answering a constant is not happening here. The merge's + ambiguous-id guard stays as defence in depth; no identity change is needed. + 4. **Hygon by proxy executes clean** on the AMD host. Its appendix lacks `card_id` / + `renderd_id` there, which is *not* a defect: it reads `/sys/module/hycu` and + `/sys/module/hydcu`, and the host runs `amdgpu`. `driver_version` is null for the same + reason. Exactly the "cannot prove a Hygon-specific value" limit recorded above. The vendor-id + edit was reverted and never committed. + + `F5` end to end, on the k3s cluster of the RTX 5090 host and on Docker 29.6.1 of the AMD host: + an unpullable image reports `state = Failed` with + `state_message = "ImagePullBackOff: Back-off pulling image ...; Error: ErrImagePull"` — the + waiting reason *plus* the appended Pod Event — and an `exits` entry whose `started_at` and + `finished_at` are empty because it never ran. A container exiting 7 reports `exit_code = 7`, + `reason = "Error"` and `state_message = "Error"`, which is the end-of-build fix: before it, + Docker left `State.Error` empty for a container that ran and exited non-zero, so the commonest + crash of all carried no reason and set no message. Its timestamps came back with six-digit + microseconds, i.e. the nanosecond truncation holds against a real daemon. + + **What C2 could not settle**, as scoped: Iluvatar, MetaX, MThreads and Cambricon have no + environment, so the MetaX temperature divisor and Cambricon's MB-means-MiB reading keep their + markers; and no MIG-enabled host existed, so `detect_output_nvidia_h100_mig.json`'s two rebuilt + cards and their compute-instance profile names stay unrefreshed. Both are recorded in the + samples README rather than guessed at. + Verify: `uv run gpustack-runtime detect --format json` on the provided host (access method + supplied by the user at that point) + +### Test Plan + +[x] I/we understand the owners of the involved components may require updates to existing tests to make +this code solid enough prior to committing the changes necessary to implement this enhancement. + +#### Prerequisite testing updates + +- `tests/gpustack_runtime/detector/fixtures/` holds only a `load()` helper and no data files. Every + detector test today is `skipif not is_supported()` and therefore never runs in CI, so each vendor + task must bring a fake binding — fake device handles plus a monkeypatch of its binding module — to + exercise `detect_info` / `detect_usage` on a machine with no driver. +- Each vendor task builds that fake **inside its own test module**, not in a shared + `fixtures/bindings.py`. The fakes have nothing to share in practice (`pynvml`, `pymxsml`, `pydcmi` + and the rest expose unrelated APIs), while one shared file would serialize seven vendor tasks whose + paths are otherwise disjoint — the coordination cost outweighs the deduplication. +- Every fake carries a **call log**, because several acceptance criteria are "issues **no** metric + call" and cannot be asserted from return values. A dozen lines per vendor. +- `pytest.ini` lists an `integration_tests` testpath that does not exist — create the directory (see + Integration tests) or drop the entry. +- Fixture `pci.ids` extract for `get_pci_device_name()` (T1) covering one AMD and one Hygon + vendor:device pair. +- Optional: add `pytest-cov` so the per-package numbers below become measurable rather than reviewed. + +#### Unit tests + +Coverage is not measured today (`pytest.ini`'s `addopts` carries no `--cov`), so these are entry targets +for the non-hardware code paths, not deltas from a baseline: + +- `gpustack_runtime/detector`: `2026-08-14` - not measured; target `>=70%` +- `gpustack_runtime/detector/pycndev`: `2026-08-14` - not measured (new); target `>=60%` (struct + layouts and error mapping; the library-loaded path stays hardware-gated) +- `gpustack_runtime/deployer` (status parsing and plan defaulting only): `2026-08-14` - not measured; + target `>=60%` + +Per-unit coverage added by task: the ABC composition and usage merge (T1); `cndev` struct layouts and +error mapping (T2); each vendor's `detect_info` / `detect_usage` split, its parity fix and the absence +of `vgpu` (T3–T10); abstractness of `detect_info` and the `--no-usage` rendering (T11); the sample +schema guard (T12); the ordering injection across all/privileged/subset/non-NVIDIA/already-declared +and all three deployers (D1); exit-status +round-tripping and backward-compatible deserialization (D2); Docker, Podman and Kubernetes exit-status +parsing plus the image-pull verdict and the Events `403` degradation (D3–D5). + +#### Integration tests + +Needing a real Docker daemon and a real Kubernetes cluster, so they live under `integration_tests/` and +are not part of `make test`'s default run. Concrete test names are added after the implementation PR +merges: + +- Docker: a workload whose run container exits non-zero reports the exit code and reason. +- Docker: a workload killed by the OOM killer reports `OOMKilled`. +- Kubernetes: a workload with an unresolvable image (`does-not-exist.invalid/x:y`) reports `Failed` + with the pull reason and the Event message. +- Kubernetes: the same workload with the `events` RBAC rule removed still reports `Failed` with the + reason, and logs rather than raises. +- Kubernetes: a healthy running workload triggers no Events request (asserted on the API call log). + +#### e2e tests + +None automated in this repository. `gpustack_runtime` is a library consumed by the GPUStack worker and +the operator, and end-to-end coverage belongs to those repositories' suites; duplicating it here would +require provisioning nine vendors' hardware in CI. The end-to-end guarantee for this change is +checkpoint `C2`: a manual pass on the environments the user provides, comparing the runtime's report +against the operator's `Devices` object on the same node and deploying one unpullable-image workload. + +## Alternatives + +- **Keep a single `detect()` with a boolean flag, no split.** Rejected: a caller that only wants a + metrics refresh would still re-run the whole inventory query, and the operator's proven + detect/monitor separation would have no counterpart. +- **Mirror the operator's names (`detect_accelerator` / `monitor_accelerator`).** Rejected: the + runtime's public surface is `detect_devices` / `Detector.detect`; renaming churns every caller for + cosmetic symmetry. +- **Copy `CUDA_DEVICE_ORDER` from the worker process's environment.** Rejected: it only works if + something upstream remembers to set it on the worker, which is the same configuration gap that + produced issue #6041 in the first place. Whether a container sees every card is something the + deployer knows and the worker does not, so the deployer decides. **Confirmed independently by the + operator**, which reached the same conclusion on its own side and declined to set the variable on + the worker Deployment — "that process runs no CUDA and propagates nothing to workloads, so the + ordering has to be stated where the injection is consumed" (`fix(devicemanager): pin CUDA's device + order where a slice's limits are read by ordinal`). Copying from the worker would therefore have + been copying a variable nobody sets. +- **Inject `CUDA_DEVICE_ORDER=PCI_BUS_ID` into every NVIDIA workload unconditionally.** Rejected: a + container given a specific device subset already has a scoped visible-devices env, and pinning the + ordering there would silently change the ordinals a caller addressing that subset relies on. +- **Introduce a generic `GPUSTACK_RUNTIME_DEPLOY_INHERIT_ENVS` passthrough list.** Rejected for now: + a new configuration surface for a single known variable. Revisit when a second one appears. +- **Report image-pull failures from `container_statuses` only, without Pod Events.** Rejected: the + waiting reason alone ("ImagePullBackOff") omits the registry error the user needs; Events carry it. +- **Keep the `vgpu` flag but stop acting on it.** Rejected: a field nothing consumes is drift waiting + to happen, and the Ascend CDI path proves consumers appear. + +## Open Questions + +- ~~Does `/dev/davinci{N}` follow the DCMI logic id or the physical id on the driver versions we + support?~~ **Answered by the end-of-build review: the physical id, and nothing stands in for it.** + The operator's `ascend/device.go` skips a device whose `GetPhysicalID` fails rather than guessing, + and so does the detector now; the CDI generator refuses to build a path without + `appendix["physical_id"]`. `C2` no longer decides this — it confirms it, by checking that the + refreshed Ascend samples carry a `physical_id` and that the node paths address the right NPUs. +- **MetaX temperature is scaled three different ways and at most one can be right.** The runtime + divides `mxSmlGetTemperatureInfo(_, TEMPERATURE_HOTSPOT)` by 100 with the comment `mC to C` — but + millidegrees to degrees is a division by 1000, and the very next line converts board power with + `// 1000 # mW to W`, so the two are inconsistent with each other. The operator applies **no** + conversion at all (`temperature = uint32(tempInfo)`). So the runtime reports the driver value ÷100 + and the operator reports it raw, which is exactly the Story 1 class of discrepancy this work exists + to remove. Found while building `T8`; deliberately left untouched, because `F1`'s MetaX row scopes + that task to the PF skip and guessing at a unit without hardware would just move the error. + **Settled by `C2`**: read one card's hotspot temperature on a real MetaX host and compare all three + readings against `mx-smi`. +- Should the usage query also be exposed at module level (e.g. `monitor_devices(devices)`) for + callers running their own polling loop, or is `Detector.detect_usage` enough? The plan ships only + `detect_devices(..., usage=...)`; add the module-level entry point when a caller needs it. +- Does the pinned `nvidia-ml-py>=13.580.65` expose `nvmlDeviceGetMemoryInfo(handle, version=2)` and + a v2 PCI-info accessor under those names, or is a version probe needed? **Answered while building + `T5`** — if the names differ, `T5` adds the probe rather than pinning a newer floor. +- ~~Is there an upstream Python `cndev` binding worth depending on instead of hand-writing + `pycndev`?~~ **Answered by `T2`: no.** The name does not exist on PyPI (checked against the full + simple index), Cambricon publishes no Python device-management binding at all — their own + `libcndev.so` consumers (`mlu-exporter`, `cambricon-k8s-device-plugin`) are Go, and `mlu-exporter` + makes the operator supply `cndev.h` themselves because the header ships with the driver. The only + third-party candidate, `mlu-api`, vendors its own `libcndev.so` (so it cannot track the host + driver's ABI), is cp310/x86_64-only and exposes device count plus core utilization and nothing + else. CNDev is a driver-versioned C ABI, i.e. the `pydcmi`/`pymxsml` situation, not the + `pynvml`/`pymtml` one — hand-written it is. diff --git a/tests/gpustack_runtime/deployer/test_docker_status.py b/tests/gpustack_runtime/deployer/test_docker_status.py new file mode 100644 index 0000000..9c517bb --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_docker_status.py @@ -0,0 +1,181 @@ +from types import SimpleNamespace + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.docker import DockerWorkloadStatus + + +def _container( + status: str, + component: str = "run", + name: str = "gpustack-test-run-0", + component_name: str = "run-0", + container_id: str = "abc123", + state: dict | None = None, + restart_policy: str = "no", + restart_count: int | None = None, + created: str = "2026-07-29T08:00:00.000000Z", +) -> SimpleNamespace: + attrs = { + "Id": container_id, + "Created": created, + "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, + "State": state if state is not None else {}, + } + if restart_count is not None: + attrs["RestartCount"] = restart_count + + return SimpleNamespace( + name=name, + status=status, + attrs=attrs, + labels={ + "runtime.gpustack.ai/component": component, + "runtime.gpustack.ai/component-name": component_name, + }, + ) + + +def test_docker_workload_status_exit_oomkilled_preserves_state(): + # A container exited 137 with OOMKilled true: the existing state verdict + # (FAILED, no restart policy) is preserved, and an exit entry carries the + # exit code plus the OOMKilled reason. + c = _container( + status="exited", + state={ + "ExitCode": 137, + "OOMKilled": True, + "Error": "", + "StartedAt": "2026-07-29T08:00:00.000000Z", + "FinishedAt": "2026-07-29T08:05:00.000000Z", + }, + restart_count=2, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "run-0" + assert exit_.token == "abc123" # noqa: S105 + assert exit_.exit_code == 137 + assert exit_.reason == "OOMKilled" + assert exit_.started_at == "2026-07-29T08:00:00.000000Z" + assert exit_.finished_at == "2026-07-29T08:05:00.000000Z" + assert exit_.restart_count == 2 + assert status.state_message == "OOMKilled" + + +def test_docker_workload_status_running_container_no_exit_entry(): + # A running container contributes no exit entry. + c = _container(status="running", state={"ExitCode": 0}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits == [] + + +def test_docker_workload_status_exit_missing_state_keys_degrades(): + # A container whose State lacks the keys degrades to an entry with the + # exit code alone rather than raising. + c = _container(status="exited", state={"ExitCode": 1}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + # Docker fills State.Error for a container that failed to start, not for one + # that ran and exited non-zero -- the commonest crash there is. The reason is + # derived from the code so this backend says what Kubernetes says. + assert exit_.reason == "Error" + assert exit_.message == "" + assert exit_.started_at == "" + assert exit_.finished_at == "" + assert exit_.restart_count == 0 + # Which is what gets the crash into the reported message at all. + assert status.state_message == "Error" + + +def test_docker_workload_status_exit_reports_no_reason_for_a_clean_exit(): + c = _container(status="exited", state={"ExitCode": 0}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].exit_code == 0 + assert status.exits[0].reason == "" + assert status.state_message == "" + + +def test_docker_workload_status_exit_ignores_gos_zero_time(): + # Docker fills an unset timestamp with Go's zero time rather than omitting + # the key, and WorkloadStatusExit documents these as empty when they never + # happened -- a UI renders the zero time as "January 1, year 1". + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "" + assert status.exits[0].finished_at == "" + + +def test_docker_workload_status_exit_truncates_nanoseconds(): + # Docker reports 9-digit nanoseconds where the Kubernetes deployer formats + # 6, and strptime's %f takes at most 6, so a consumer parsing this field + # would succeed on one backend and raise on the other. + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "2026-07-29T08:00:00.123456789Z", + "FinishedAt": "2026-07-29T08:05:00.987654321Z", + }, + ) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "2026-07-29T08:00:00.123456Z" + assert status.exits[0].finished_at == "2026-07-29T08:05:00.987654Z" + + +def test_docker_workload_status_exit_covers_a_restarting_container(): + # A crash-looping container sits in "restarting" between attempts with the + # last attempt's exit code already filled in, which is the diagnosis being + # asked for: whether the user gets it must not depend on when the poll lands. + c = _container(status="restarting", state={"ExitCode": 1}) + + status = DockerWorkloadStatus(name="test", d_containers=[c]) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 1 + + +def test_docker_workload_status_exit_includes_init_containers(): + # Init containers get an exit entry too, not just run containers. + init_c = _container( + status="exited", + component="init", + name="gpustack-test-init-0", + component_name="init-0", + container_id="init-abc", + state={"ExitCode": 1, "Error": "context deadline exceeded"}, + ) + run_c = _container(status="running", state={}) + + status = DockerWorkloadStatus(name="test", d_containers=[init_c, run_c]) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "init-0" + assert exit_.token == "init-abc" # noqa: S105 + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "context deadline exceeded" diff --git a/tests/gpustack_runtime/deployer/test_kubernetes_status.py b/tests/gpustack_runtime/deployer/test_kubernetes_status.py new file mode 100644 index 0000000..c46a443 --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_kubernetes_status.py @@ -0,0 +1,446 @@ +# The cases below drive the Kubernetes deployer's own status parsing, which is +# where the exit list, the image-pull verdict and the Pod Events read become +# observable without a live cluster: hand-built kubernetes.client objects plus a +# fake CoreV1Api carrying a call log. + +from datetime import datetime, timezone + +import kubernetes.client +import pytest + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.kuberentes import KubernetesWorkloadStatus + +_REGISTRY_MESSAGE = 'Back-off pulling image "does-not-exist.invalid/x:y"' +_EVENT_MESSAGE = ( + 'Failed to pull image "does-not-exist.invalid/x:y": ' + "failed to resolve reference: unauthorized" +) + + +class _FakeCoreV1Api: + """ + Stand-in for the Kubernetes CoreV1Api. + + Every call is recorded, so a test can assert that an Events read did *not* + happen - which cannot be observed from the parsed status alone. + """ + + def __init__( + self, + events: list[kubernetes.client.CoreV1Event] | None = None, + raises: Exception | None = None, + ): + self.calls: list[tuple[str, dict]] = [] + self._events = events or [] + self._raises = raises + + def list_namespaced_event(self, **kwargs): + self.calls.append(("list_namespaced_event", kwargs)) + if self._raises: + raise self._raises + return kubernetes.client.CoreV1EventList(items=self._events) + + +def _event( + message: str = _EVENT_MESSAGE, + type_: str = "Warning", + reason: str = "Failed", + last_timestamp: datetime | None = None, +) -> kubernetes.client.CoreV1Event: + return kubernetes.client.CoreV1Event( + metadata=kubernetes.client.V1ObjectMeta(name="gpustack-test.1"), + involved_object=kubernetes.client.V1ObjectReference( + kind="Pod", + name="gpustack-test", + namespace="default", + ), + message=message, + reason=reason, + type=type_, + last_timestamp=last_timestamp, + ) + + +def _pod( + phase: str, + container_statuses: list[kubernetes.client.V1ContainerStatus] | None = None, + init_container_statuses: list[kubernetes.client.V1ContainerStatus] | None = None, + init_containers: list[kubernetes.client.V1Container] | None = None, + message: str | None = None, + annotations: dict[str, str] | None = None, + container_names: list[str] | None = None, + uid: str | None = "gpustack-test-uid", +) -> kubernetes.client.V1Pod: + return kubernetes.client.V1Pod( + metadata=kubernetes.client.V1ObjectMeta( + name="gpustack-test", + namespace="default", + uid=uid, + creation_timestamp=datetime(2026, 8, 14, 8, 0, 0, tzinfo=timezone.utc), + labels={"app": "test"}, + annotations=annotations, + ), + spec=kubernetes.client.V1PodSpec( + containers=[ + kubernetes.client.V1Container(name=n) + for n in (container_names or ["run-0"]) + ], + init_containers=init_containers, + ), + status=kubernetes.client.V1PodStatus( + phase=phase, + message=message, + container_statuses=container_statuses, + init_container_statuses=init_container_statuses, + ), + ) + + +def _waiting_status( + name: str = "run-0", + reason: str = "ImagePullBackOff", + message: str = _REGISTRY_MESSAGE, + restart_count: int = 0, +) -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="does-not-exist.invalid/x:y", + image_id="", + ready=False, + restart_count=restart_count, + state=kubernetes.client.V1ContainerState( + waiting=kubernetes.client.V1ContainerStateWaiting( + reason=reason, + message=message, + ), + ), + ) + + +def _terminated_status( + name: str = "run-0", + exit_code: int = 1, + reason: str = "Error", + restart_count: int = 2, +) -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=False, + restart_count=restart_count, + state=kubernetes.client.V1ContainerState( + terminated=kubernetes.client.V1ContainerStateTerminated( + exit_code=exit_code, + reason=reason, + message="the process exited", + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + finished_at=datetime(2026, 8, 14, 8, 2, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def _restarted_status( + name: str = "run-0", +) -> kubernetes.client.V1ContainerStatus: + # A container that has been restarted after an OOM kill: the current state + # is running, the termination only survives in last_state. + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=True, + restart_count=1, + state=kubernetes.client.V1ContainerState( + running=kubernetes.client.V1ContainerStateRunning( + started_at=datetime(2026, 8, 14, 8, 3, 0, tzinfo=timezone.utc), + ), + ), + last_state=kubernetes.client.V1ContainerState( + terminated=kubernetes.client.V1ContainerStateTerminated( + exit_code=137, + reason="OOMKilled", + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + finished_at=datetime(2026, 8, 14, 8, 2, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def _ready_status(name: str = "run-0") -> kubernetes.client.V1ContainerStatus: + return kubernetes.client.V1ContainerStatus( + name=name, + image="gpustack/runner:latest", + image_id="", + ready=True, + restart_count=0, + state=kubernetes.client.V1ContainerState( + running=kubernetes.client.V1ContainerStateRunning( + started_at=datetime(2026, 8, 14, 8, 1, 0, tzinfo=timezone.utc), + ), + ), + ) + + +def test_image_pull_backoff_reports_failed_with_event_message(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + # A Pod stuck on an unpullable image has failed, it is not pending. + assert status.state == WorkloadStatusStateEnum.FAILED + # The waiting reason alone omits the registry error, so the Event message + # is appended to it. + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + assert _EVENT_MESSAGE in status.state_message + + assert len(status.exits) == 1 + assert status.exits[0].name == "run-0" + assert status.exits[0].token == "run-0" # noqa: S105 + assert status.exits[0].exit_code is None + assert status.exits[0].reason == "ImagePullBackOff" + assert status.exits[0].message == _REGISTRY_MESSAGE + + # Exactly one Events read, field-selected to this Pod. The UID is part of the + # selector because a recreated Pod takes the same name. + assert [c[0] for c in api.calls] == ["list_namespaced_event"] + assert api.calls[0][1]["namespace"] == "default" + assert api.calls[0][1]["field_selector"] == ( + "involvedObject.name=gpustack-test,involvedObject.uid=gpustack-test-uid" + ) + + +def test_pod_event_message_takes_the_latest_by_timestamp(): + # The API returns Events in no particular order, so the newest one is not + # necessarily last in the list. + stale = _event( + message="0/3 nodes are available: insufficient nvidia.com/gpu.", + reason="FailedScheduling", + last_timestamp=datetime(2026, 8, 14, 7, 0, 0, tzinfo=timezone.utc), + ) + latest = _event(last_timestamp=datetime(2026, 8, 14, 9, 0, 0, tzinfo=timezone.utc)) + api = _FakeCoreV1Api(events=[latest, stale]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert _EVENT_MESSAGE in status.state_message + assert "FailedScheduling" not in status.state_message + + +def test_pod_event_message_prefers_the_image_pull_failure(): + # A warning about something else, stamped later than the pull failure, must + # not stand in as the diagnosis of an image pull that cannot succeed. + later_unrelated = _event( + message="Liveness probe failed: connection refused", + reason="Unhealthy", + last_timestamp=datetime(2026, 8, 14, 10, 0, 0, tzinfo=timezone.utc), + ) + pull_failure = _event( + last_timestamp=datetime(2026, 8, 14, 9, 0, 0, tzinfo=timezone.utc), + ) + api = _FakeCoreV1Api(events=[pull_failure, later_unrelated]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert _EVENT_MESSAGE in status.state_message + assert "Liveness probe" not in status.state_message + + +@pytest.mark.parametrize( + "reason", + [ + "ErrImageNeverPull", + "ErrImagePull", + "ImagePullBackOff", + "InvalidImageName", + "RegistryUnavailable", + ], +) +def test_image_pull_blocked_reasons_report_failed(reason): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status(reason=reason)]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert reason in status.state_message + + +def test_pending_on_another_reason_stays_pending(): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + container_statuses=[ + _waiting_status(reason="ContainerCreating", message=""), + ], + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.PENDING + # Not a blocked Pod, so no Events traffic. + assert api.calls == [] + + +def test_terminated_container_reports_exit_code(): + api = _FakeCoreV1Api() + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Failed", container_statuses=[_terminated_status()]), + core_api=api, + ) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "the process exited" + assert exit_.started_at == "2026-08-14T08:01:00.000000Z" + assert exit_.finished_at == "2026-08-14T08:02:00.000000Z" + assert exit_.restart_count == 2 + # Events are read for blocked Pods only. + assert api.calls == [] + + +def test_restarted_container_reports_last_terminated_state(): + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Running", container_statuses=[_restarted_status()]), + ) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 137 + assert status.exits[0].reason == "OOMKilled" + assert status.exits[0].restart_count == 1 + + +def test_events_forbidden_degrades_to_state_and_reason(): + api = _FakeCoreV1Api( + raises=kubernetes.client.exceptions.ApiException(status=403), + ) + + # A cluster that has not applied the events RBAC rule must keep working. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + assert _EVENT_MESSAGE not in status.state_message + assert status.exits[0].reason == "ImagePullBackOff" + + +def test_running_pod_makes_no_events_call(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Running", container_statuses=[_ready_status()]), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.RUNNING + assert status.exits == [] + assert api.calls == [] + + +def test_running_pod_with_blocked_sidecar_reports_exit_without_events_call(): + api = _FakeCoreV1Api(events=[_event()]) + + # The verdict stays readiness-based - a serving Pod does not fail because a + # sidecar cannot pull - but the blocked container is still reported. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Running", + container_statuses=[_ready_status(), _waiting_status(name="run-1")], + container_names=["run-0", "run-1"], + ), + ) + status_with_api = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Running", + container_statuses=[_ready_status(), _waiting_status(name="run-1")], + container_names=["run-0", "run-1"], + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.INITIALIZING + assert [e.reason for e in status.exits] == ["ImagePullBackOff"] + assert status_with_api.state == status.state + assert api.calls == [] + + +def test_blocked_init_container_reports_failed_with_component_name(): + api = _FakeCoreV1Api(events=[_event()]) + + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + init_containers=[kubernetes.client.V1Container(name="init-0")], + init_container_statuses=[_waiting_status(name="init-0")], + annotations={"runtime.gpustack.ai/component-init-0-name": "downloader"}, + ), + core_api=api, + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + assert status.exits[0].name == "downloader" + assert status.exits[0].token == "init-0" # noqa: S105 + assert status.exits[0].reason == "ImagePullBackOff" + assert [c[0] for c in api.calls] == ["list_namespaced_event"] + + +def test_blocked_pod_without_api_client_still_reports_failed(): + # The pre-manifest path, and any call site that has no client at hand. + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod("Pending", container_statuses=[_waiting_status()]), + ) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert "ImagePullBackOff" in status.state_message + assert _REGISTRY_MESSAGE in status.state_message + + +def test_pull_diagnosis_overrides_pod_status_message(): + status = KubernetesWorkloadStatus( + name="test", + k_pod=_pod( + "Pending", + container_statuses=[_waiting_status()], + message="some generic pod message", + ), + ) + + assert "some generic pod message" not in status.state_message + assert "ImagePullBackOff" in status.state_message diff --git a/tests/gpustack_runtime/deployer/test_podman_status.py b/tests/gpustack_runtime/deployer/test_podman_status.py new file mode 100644 index 0000000..f3cbcdd --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_podman_status.py @@ -0,0 +1,144 @@ +from types import SimpleNamespace + +from gpustack_runtime.deployer.__types__ import WorkloadStatusStateEnum +from gpustack_runtime.deployer.podman import PodmanWorkloadStatus + + +def _container( + status: str, + component: str = "run", + name: str = "gpustack-test-run-0", + component_name: str = "run-0", + container_id: str = "abc123", + state: dict | None = None, + restart_policy: str = "no", + restart_count: int | None = None, + created: str = "2026-07-29T08:00:00.000000Z", +) -> SimpleNamespace: + attrs = { + "Id": container_id, + "Created": created, + "HostConfig": {"RestartPolicy": {"Name": restart_policy}}, + "State": state if state is not None else {}, + } + if restart_count is not None: + attrs["RestartCount"] = restart_count + + return SimpleNamespace( + name=name, + status=status, + attrs=attrs, + labels={ + "runtime.gpustack.ai/component": component, + "runtime.gpustack.ai/component-name": component_name, + }, + ) + + +def test_podman_workload_status_exit_oomkilled_preserves_state(): + # A container exited 137 with OOMKilled true: the existing state verdict + # (FAILED, no restart policy) is preserved, and an exit entry carries the + # exit code plus the OOMKilled reason. + c = _container( + status="exited", + state={ + "ExitCode": 137, + "OOMKilled": True, + "Error": "", + "StartedAt": "2026-07-29T08:00:00.000000Z", + "FinishedAt": "2026-07-29T08:05:00.000000Z", + }, + restart_count=2, + ) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "run-0" + assert exit_.token == "abc123" # noqa: S105 + assert exit_.exit_code == 137 + assert exit_.reason == "OOMKilled" + assert exit_.started_at == "2026-07-29T08:00:00.000000Z" + assert exit_.finished_at == "2026-07-29T08:05:00.000000Z" + assert exit_.restart_count == 2 + assert status.state_message == "OOMKilled" + + +def test_podman_workload_status_running_container_no_exit_entry(): + # A running container contributes no exit entry. + c = _container(status="running", state={"ExitCode": 0}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits == [] + + +def test_podman_workload_status_exit_missing_state_keys_degrades(): + # A container whose State lacks the keys degrades to an entry with the + # exit code alone rather than raising. + c = _container(status="exited", state={"ExitCode": 1}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.state == WorkloadStatusStateEnum.FAILED + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.exit_code == 1 + # The State is Docker-compatible, so both backends derive the reason from the + # code and agree with what Kubernetes reports for the same event. + assert exit_.reason == "Error" + assert exit_.message == "" + assert exit_.started_at == "" + assert exit_.finished_at == "" + assert exit_.restart_count == 0 + assert status.state_message == "Error" + + +def test_podman_workload_status_exit_ignores_gos_zero_time(): + c = _container( + status="exited", + state={ + "ExitCode": 1, + "StartedAt": "0001-01-01T00:00:00Z", + "FinishedAt": "0001-01-01T00:00:00Z", + }, + ) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert status.exits[0].started_at == "" + assert status.exits[0].finished_at == "" + + +def test_podman_workload_status_exit_covers_a_restarting_container(): + c = _container(status="restarting", state={"ExitCode": 1}) + + status = PodmanWorkloadStatus(name="test", d_containers=[c]) + + assert len(status.exits) == 1 + assert status.exits[0].exit_code == 1 + + +def test_podman_workload_status_exit_includes_init_containers(): + # Init containers get an exit entry too, not just run containers. + init_c = _container( + status="exited", + component="init", + name="gpustack-test-init-0", + component_name="init-0", + container_id="init-abc", + state={"ExitCode": 1, "Error": "context deadline exceeded"}, + ) + run_c = _container(status="running", state={}) + + status = PodmanWorkloadStatus(name="test", d_containers=[init_c, run_c]) + + assert len(status.exits) == 1 + exit_ = status.exits[0] + assert exit_.name == "init-0" + assert exit_.token == "init-abc" # noqa: S105 + assert exit_.exit_code == 1 + assert exit_.reason == "Error" + assert exit_.message == "context deadline exceeded" diff --git a/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py b/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py new file mode 100644 index 0000000..4001a2c --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_visible_devices_ordering.py @@ -0,0 +1,349 @@ +# The cases below drive the deployers' own plan-to-container conversion, +# which is where the device ordering pin becomes observable without a live +# Docker daemon, Podman socket or Kubernetes cluster. +# ruff: noqa: SLF001 + +from types import SimpleNamespace + +import docker.errors +import kubernetes.client +import podman.errors +import pytest + +from gpustack_runtime import envs +from gpustack_runtime.deployer.__types__ import ( + Container, + ContainerEnv, + ContainerExecution, + ContainerResources, + Deployer, + DevicesMaterial, +) +from gpustack_runtime.deployer.docker import DockerDeployer, DockerWorkloadPlan +from gpustack_runtime.deployer.kuberentes import ( + KubernetesDeployer, + KubernetesWorkloadPlan, +) +from gpustack_runtime.deployer.podman import PodmanDeployer, PodmanWorkloadPlan +from gpustack_runtime.detector import ManufacturerEnum + +_ORDERING_ENV = "CUDA_DEVICE_ORDER" + +_NVIDIA_MATERIALS = { + "NVIDIA_VISIBLE_DEVICES": DevicesMaterial( + manufacturer=ManufacturerEnum.NVIDIA, + runtime_env="NVIDIA_VISIBLE_DEVICES", + backend_env=["CUDA_VISIBLE_DEVICES"], + cdi="nvidia.com/gpu", + runtime_values={"0": "0", "1": "1"}, + backend_values={"CUDA_VISIBLE_DEVICES": {"0": "0", "1": "1"}}, + ), +} +_AMD_MATERIALS = { + "AMD_VISIBLE_DEVICES": DevicesMaterial( + manufacturer=ManufacturerEnum.AMD, + runtime_env="AMD_VISIBLE_DEVICES", + backend_env=["HIP_VISIBLE_DEVICES"], + cdi="amd.com/gpu", + runtime_values={"0": "0"}, + backend_values={"HIP_VISIBLE_DEVICES": {"0": "0"}}, + ), +} +_MIXED_MATERIALS = {**_NVIDIA_MATERIALS, **_AMD_MATERIALS} + + +def _deployer(cls, materials: dict[str, DevicesMaterial]): + # Bypass the deployer's __init__, which reaches out to a live daemon or + # API server, and pin the materials so _prepare() short-circuits instead + # of detecting devices. + deployer = object.__new__(cls) + Deployer.__init__(deployer, "test") + deployer._materials = materials + return deployer + + +def _container( + resources: dict, + privileged: bool = False, + declared_envs: dict[str, str] | None = None, +) -> Container: + container_resources = ContainerResources() + container_resources.update(resources) + return Container( + name="default", + image="gpustack/runner:latest", + execution=ContainerExecution(privileged=privileged), + envs=[ContainerEnv(name=n, value=v) for n, v in (declared_envs or {}).items()], + resources=container_resources, + ) + + +class _FakeContainers: + """ + Stand-in for the Docker/Podman clients' container collection. + """ + + def __init__(self, not_found, created: list[dict]): + self._not_found = not_found + self._created = created + + def get(self, name): + raise self._not_found(name) + + def create(self, **kwargs): + self._created.append(kwargs) + return SimpleNamespace(name=kwargs.get("name")) + + +def _docker_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, +) -> list[tuple[str, str]]: + # Keep the requested devices on the env injection path, + # so no CDI specification is generated. + monkeypatch.setattr( + envs, + "GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY", + "Env", + ) + + created: list[dict] = [] + deployer = _deployer(DockerDeployer, materials) + deployer._client = SimpleNamespace( + containers=_FakeContainers(docker.errors.NotFound, created), + ) + deployer._get_image = lambda *_args, **_kwargs: container.image + deployer._mutate_create_options = lambda create_options: create_options + + workload = DockerWorkloadPlan(name="test", containers=[container]) + workload.validate_and_default() + deployer._create_containers(workload, {}, SimpleNamespace(id="pause")) + + return list(created[0]["environment"].items()) + + +def _podman_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, +) -> list[tuple[str, str]]: + # Podman always requests devices via CDI, + # skip generating the CDI specification. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_PODMAN_CDI_SPECS_GENERATE", False) + + created: list[dict] = [] + deployer = _deployer(PodmanDeployer, materials) + deployer._client = SimpleNamespace( + containers=_FakeContainers(podman.errors.NotFound, created), + ) + deployer._get_image = lambda *_args, **_kwargs: container.image + deployer._mutate_create_options = lambda create_options: create_options + + workload = PodmanWorkloadPlan(name="test", containers=[container]) + workload.validate_and_default() + deployer._create_containers(workload, {}, SimpleNamespace(id="pause")) + + return list(created[0]["environment"].items()) + + +def _kubernetes_container_envs( + materials: dict[str, DevicesMaterial], + container: Container, + monkeypatch, + policy: str = "env", +) -> list[tuple[str, str]]: + monkeypatch.setattr( + "gpustack_runtime.deployer.kuberentes.get_resource_injection_policy", + lambda: policy, + ) + # Resolving the RuntimeClass reads the cluster. + monkeypatch.setattr( + "gpustack_runtime.deployer.kuberentes._resolve_runtime_class_name", + lambda *_args: None, + ) + + class _FakeCoreV1Api: + def __init__(self, client=None): + pass + + def read_namespaced_pod(self, name, namespace): + raise kubernetes.client.exceptions.ApiException(status=404) + + def create_namespaced_pod(self, namespace, body): + return body + + monkeypatch.setattr(kubernetes.client, "CoreV1Api", _FakeCoreV1Api) + + deployer = _deployer(KubernetesDeployer, materials) + deployer._client = None + deployer._node_name = None + deployer._image_pull_secret = None + deployer._mutate_create_pod = lambda pod: pod + + workload = KubernetesWorkloadPlan( + name="test", + namespace="default", + containers=[container], + ) + workload.validate_and_default() + pod = deployer._create_pod(workload, {}) + + return [(e.name, e.value) for e in pod.spec.containers[0].env] + + +@pytest.mark.parametrize( + "name, materials, runtime_envs, expected", + [ + ( + "NVIDIA", + _NVIDIA_MATERIALS, + ["NVIDIA_VISIBLE_DEVICES"], + {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}, + ), + ( + "non-NVIDIA", + _AMD_MATERIALS, + ["AMD_VISIBLE_DEVICES"], + {}, + ), + ( + "mixed manufacturers", + _MIXED_MATERIALS, + ["NVIDIA_VISIBLE_DEVICES", "AMD_VISIBLE_DEVICES"], + {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}, + ), + ( + "unknown runtime visible devices env", + _NVIDIA_MATERIALS, + ["UNKNOWN_RUNTIME_VISIBLE_DEVICES"], + {}, + ), + ( + "no runtime visible devices env", + _NVIDIA_MATERIALS, + [], + {}, + ), + ], +) +def test_map_visible_devices_ordering(name, materials, runtime_envs, expected): + deployer = _deployer(DockerDeployer, materials) + actual = deployer.map_visible_devices_ordering(runtime_envs) + assert actual == expected, f"case {name} expected {expected}, but got {actual}" + + +@pytest.mark.parametrize( + "runner", + [ + _docker_container_envs, + _podman_container_envs, + _kubernetes_container_envs, + ], + ids=["docker", "podman", "kubernetes"], +) +@pytest.mark.parametrize( + "name, materials, resources, privileged, declared_envs, expected", + [ + ( + "all devices", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "all"}, + False, + None, + ["PCI_BUS_ID"], + ), + ( + "specific devices, privileged", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "0"}, + True, + None, + ["PCI_BUS_ID"], + ), + ( + "specific devices, unprivileged", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "0"}, + False, + None, + [], + ), + ( + "all devices, non-NVIDIA", + _AMD_MATERIALS, + {"amd.com/devices": "all"}, + False, + None, + [], + ), + ( + "all devices, ordering declared by the container", + _NVIDIA_MATERIALS, + {"nvidia.com/devices": "all"}, + False, + {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"}, + ["FASTEST_FIRST"], + ), + ( + "all devices, auto-mapped on a mixed-manufacturer node", + _MIXED_MATERIALS, + {"gpustack.ai/devices": "all"}, + False, + None, + ["PCI_BUS_ID"], + ), + ], +) +def test_visible_devices_ordering_injection( + name, + materials, + resources, + privileged, + declared_envs, + expected, + runner, + monkeypatch, +): + container_envs = runner( + materials, + _container(resources, privileged, declared_envs), + monkeypatch, + ) + actual = [v for n, v in container_envs if n == _ORDERING_ENV] + assert actual == expected, f"case {name} expected {expected}, but got {actual}" + + +@pytest.mark.parametrize( + "name, resources, expected", + [ + ( + "all devices", + {"nvidia.com/devices": "all"}, + ["PCI_BUS_ID"], + ), + ( + "specific devices", + {"nvidia.com/devices": "0"}, + [], + ), + ], +) +def test_visible_devices_ordering_injection_with_device_plugin( + name, + resources, + expected, + monkeypatch, +): + # Under the KDP injection policy the devices are allocated by a device + # plugin, so a privileged request loses its privilege: an "all" request + # still receives every card and gets the ordering pinned, a specific + # request does not. + container_envs = _kubernetes_container_envs( + _NVIDIA_MATERIALS, + _container(resources, privileged=True), + monkeypatch, + policy="kdp", + ) + actual = [v for n, v in container_envs if n == _ORDERING_ENV] + assert actual == expected, f"case {name} expected {expected}, but got {actual}" diff --git a/tests/gpustack_runtime/deployer/test_workload_status.py b/tests/gpustack_runtime/deployer/test_workload_status.py index e2eb906..3e92068 100644 --- a/tests/gpustack_runtime/deployer/test_workload_status.py +++ b/tests/gpustack_runtime/deployer/test_workload_status.py @@ -1,9 +1,14 @@ +import json from datetime import datetime, timezone import kubernetes.client from gpustack_runtime import envs -from gpustack_runtime.deployer.__types__ import WorkloadStatus +from gpustack_runtime.deployer.__types__ import ( + WorkloadStatus, + WorkloadStatusExit, + WorkloadStatusStateEnum, +) from gpustack_runtime.deployer.kuberentes import ( KubernetesWorkloadStatus, _pin_pod_for_kueue, @@ -63,6 +68,76 @@ def test_workload_status_annotations_json_roundtrip(): assert restored.annotations == status.annotations +def test_workload_status_exits_default_empty(): + # Deployers that report no terminated container leave the list empty, + # so consumers can iterate it without a None check. + status = WorkloadStatus(name="test", created_at="2026-07-29T08:00:00.000000Z") + assert status.exits == [] + + +def test_workload_status_no_exits_json_roundtrip(): + status = WorkloadStatus(name="test", created_at="2026-07-29T08:00:00.000000Z") + restored = WorkloadStatus.from_json(status.to_json()) + assert restored.exits == [] + + +def test_workload_status_exits_json_roundtrip(): + exits = [ + # A terminated container, as Docker/Podman report it. + WorkloadStatusExit( + name="run-0", + token="a1b2c3", # noqa: S106 + exit_code=137, + reason="OOMKilled", + message="", + started_at="2026-07-29T08:00:00.000000Z", + finished_at="2026-07-29T08:05:00.000000Z", + restart_count=2, + ), + # A container blocked from starting, as Kubernetes reports it: + # a reason, and no exit code at all. + WorkloadStatusExit( + name="run-1", + token="run-1", # noqa: S106 + reason="ImagePullBackOff", + message='Back-off pulling image "does-not-exist.invalid/x:y"', + ), + ] + status = WorkloadStatus( + name="test", + created_at="2026-07-29T08:00:00.000000Z", + exits=exits, + ) + + restored = WorkloadStatus.from_json(status.to_json()) + + assert restored.exits == exits + assert all(isinstance(e, WorkloadStatusExit) for e in restored.exits) + assert restored.exits[1].exit_code is None + + +def test_workload_status_exits_absent_in_legacy_payload(): + # A payload serialized before the exit list existed must keep deserializing. + legacy = json.dumps( + { + "name": "test", + "created_at": "2026-07-29T08:00:00.000000Z", + "namespace": None, + "labels": {}, + "annotations": {}, + "state_message": "", + "executable": [], + "loggable": [], + "state": "Unknown", + }, + ) + + restored = WorkloadStatus.from_json(legacy) + + assert restored.exits == [] + assert restored.state == WorkloadStatusStateEnum.UNKNOWN + + def _pinning_pod(labels=None) -> kubernetes.client.V1Pod: return kubernetes.client.V1Pod( metadata=kubernetes.client.V1ObjectMeta(name="p", labels=labels or {}), diff --git a/tests/gpustack_runtime/detector/samples/README.md b/tests/gpustack_runtime/detector/samples/README.md index ed5c025..4a24e37 100644 --- a/tests/gpustack_runtime/detector/samples/README.md +++ b/tests/gpustack_runtime/detector/samples/README.md @@ -1,3 +1,46 @@ # GPUStack Runtime Detector Samples This directory contains output samples fetched by the GPUStack Runtime Detector. + +`test_samples.py` guards the shape of every `detect_output_*.json` against the `Device` contract. + +## Captured from real hardware + +These were captured with `gpustack-runtime detect --format json` / `topology --format json` on the +host named, so their values are measurements rather than hand-alignments: + +| Sample | Host | +| --- | --- | +| `*_amd_rx7800xt.json` | 2 × AMD Radeon RX 7800 XT, ROCm 7.2.0, driver 6.16.13 | +| `*_ascend_910b2.json` | 8 × Ascend 910B2 | +| `*_nvidia_rtx4090_48g.json` | 2 × RTX 4090 48 GB, driver 595.84 | +| `*_nvidia_rtx5090d.json` | 1 × RTX 5090 D, driver 595.84 | +| `*_thead_ppu.json` | 16 × PPU-ZW810E | + +Two facts worth reading off them, because both decide a device-node path: + +- **Ascend carries `appendix["physical_id"]` on every device.** `/dev/davinciN` is numbered by it, and + a device whose physical id cannot be read is dropped rather than addressed by the logic id. On this + host the two happen to agree (0–7). +- **T-Head's `appendix["minor_number"]` is not its enumeration index.** They run one apart on this host + (index 3 → minor 4, index 15 → minor 16) because `/dev/alixpu` holds minor 0 of the same + character-device major, but that offset is an observation about this host and driver, not a rule — + neither number is ever computed from the other. `/dev/alixpu_ppu` is named by the *card ordinal*; + the minor is the card's identity, and the CDI generator compares it against the node's own minor to + prove the ordinal reached the right card. + +## Not captured from real hardware + +The remaining samples are hand-aligned and carry the caveats below. + +In `detect_output_nvidia_h100_mig.json`, the two MIG-enabled cards (index 0 and 3) are rebuilt by +hand from the instances that the retired standalone-MIG shape reported in their place, so they carry +only the fields those instances copied off the card: `name`, `uuid`, `cores`, `memory`, the usage +fields and the fabric keys are absent because the old shape did not record them, not because the +driver had no answer. Card 3's instances are still named after their compute-instance profile +(`1c.3g.40gb`), where the detector now reports the GPU-instance profile. Refresh from real hardware +before relying on either — no MIG-enabled host was available. + +`detect_output_hygon_k100ai.json` is likewise unrefreshed. The Hygon *code path* was exercised on the +AMD host by pointing its PCI vendor gate at `0x1002`, which proves the path runs and its field names +resolve, but the values it produced are an AMD card's and were deliberately not written here. diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json index bf37d61..2e4c822 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi300x.json @@ -19,7 +19,6 @@ "power_used": 132, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:26:00.0", "numa": "0", "card_id": 9, @@ -46,7 +45,6 @@ "power_used": 141, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:46:00.0", "numa": "0", "card_id": 17, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json index d0f1132..777c301 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_mi308x.json @@ -19,7 +19,6 @@ "power_used": 166, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:25:00.0", "numa": "0", "card_id": 0, @@ -46,7 +45,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:2a:00.0", "numa": "0", "card_id": 8, @@ -73,7 +71,6 @@ "power_used": 162, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:45:00.0", "numa": "0", "card_id": 16, @@ -100,7 +97,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:4a:00.0", "numa": "0", "card_id": 24, @@ -127,7 +123,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:a5:00.0", "numa": "1", "card_id": 32, @@ -154,7 +149,6 @@ "power_used": 162, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:a8:00.0", "numa": "1", "card_id": 40, @@ -181,7 +175,6 @@ "power_used": 160, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:c5:00.0", "numa": "1", "card_id": 48, @@ -208,7 +201,6 @@ "power_used": 158, "appendix": { "arch_family": "Arctic Islands", - "vgpu": false, "bdf": "0000:c8:00.0", "numa": "1", "card_id": 56, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json b/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json index 6207deb..f41cda4 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_amd_rx7800xt.json @@ -2,28 +2,53 @@ { "manufacturer": "amd", "index": 0, - "name": "Navi 32 [Radeon RX 7700 XT / 7800 XT]", + "name": "AMD Radeon RX 7800 XT", "uuid": "GPU-5c88007d760374f3", - "driver_version": "6.12.12", - "runtime_version": "7.1", - "runtime_version_original": "7.1.1", + "driver_version": "6.16.13", + "runtime_version": "7.2", + "runtime_version_original": "7.2.0", "compute_capability": "gfx1101", "cores": 60, - "cores_utilization": 19, + "cores_utilization": 13, "memory": 16368, - "memory_used": 206, - "memory_utilization": 1.26, + "memory_used": 174, + "memory_utilization": 1.06, "memory_status": "healthy", - "temperature": 34, + "temperature": 43, "power": 236, - "power_used": 17, + "power_used": 10, "appendix": { "arch_family": "GC 11.0.0", - "vgpu": false, - "bdf": "0000:03:00.0", + "bdf": "0000:04:00.0", "numa": "0", "card_id": 1, "renderd_id": 128 } + }, + { + "manufacturer": "amd", + "index": 1, + "name": "AMD Radeon RX 7800 XT", + "uuid": "GPU-d99e7fe92c7bdf75", + "driver_version": "6.16.13", + "runtime_version": "7.2", + "runtime_version_original": "7.2.0", + "compute_capability": "gfx1101", + "cores": 60, + "cores_utilization": 34, + "memory": 16368, + "memory_used": 174, + "memory_utilization": 1.06, + "memory_status": "healthy", + "temperature": 44, + "power": 212, + "power_used": 10, + "appendix": { + "arch_family": "GC 11.0.0", + "bdf": "0000:07:00.0", + "numa": "0", + "card_id": 0, + "renderd_id": 129 + } } ] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json b/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json index b6d0254..5e7291c 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_ascend_310p3.json @@ -19,7 +19,6 @@ "power_used": null, "appendix": { "arch_family": "Ascend310P3", - "vgpu": false, "bdf": "0000:81:00.0", "numa": "2", "card_id": 4, @@ -47,7 +46,6 @@ "power_used": null, "appendix": { "arch_family": "Ascend310P3", - "vgpu": false, "bdf": "0000:81:00.0", "numa": "2", "card_id": 4, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json b/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json index d433463..85a680f 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_ascend_910b2.json @@ -3,248 +3,248 @@ "manufacturer": "ascend", "index": 0, "name": "910B2", - "uuid": "7281A664 C0C977 2879FD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "E0F4EE64 802061B1 6A691492 89528485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3421, - "memory_utilization": 5.22, + "memory_used": 3432, + "memory_utilization": 5.24, "memory_status": "healthy", - "temperature": 49, + "temperature": 47, "power": null, - "power_used": 99.9, + "power_used": 101.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:c1:00.0", - "numa": "6", "card_id": 0, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.6", + "physical_id": 0, + "numa": "6", + "roce_ip": "10.52.32.11", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 1, "name": "910B2", - "uuid": "C0DBA664 C107DB 2A9F372 DFD00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "E281A66C 140C979 2CFBED72 A4500485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 12458, - "memory_utilization": 19.01, + "memory_used": 3427, + "memory_utilization": 5.23, "memory_status": "healthy", - "temperature": 51, + "temperature": 45, "power": null, - "power_used": 100.0, + "power_used": 101.0, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:01:00.0", - "numa": "0", "card_id": 1, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.5", + "physical_id": 1, + "numa": "0", + "roce_ip": "10.52.32.10", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 2, "name": "910B2", - "uuid": "6EC1A664 100D35B 6BB27372 DFD00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "7281A664 1003374 3444AB72 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3413, - "memory_utilization": 5.21, + "memory_used": 3420, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 49, + "temperature": 45, "power": null, - "power_used": 100.2, + "power_used": 96.2, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:c2:00.0", - "numa": "6", "card_id": 2, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.8", + "physical_id": 2, + "numa": "6", + "roce_ip": "10.52.32.13", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 3, "name": "910B2", - "uuid": "6281A66C 80F7B1 11DAB172 99D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "687DA66C 100D2DF 1915AB72 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3412, - "memory_utilization": 5.21, + "memory_used": 3421, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 100.3, + "power_used": 97.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:02:00.0", - "numa": "0", "card_id": 3, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.7", + "physical_id": 3, + "numa": "0", + "roce_ip": "10.52.32.12", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 4, "name": "910B2", - "uuid": "6281A66C 120F0F3 1BD3EB72 86D28485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "D0DBA664 12066DD 27A93572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3410, - "memory_utilization": 5.2, + "memory_used": 3421, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 49, + "temperature": 44, "power": null, - "power_used": 102.2, + "power_used": 95.0, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:81:00.0", - "numa": "4", "card_id": 4, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.11", + "physical_id": 4, + "numa": "4", + "roce_ip": "10.52.32.16", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 5, "name": "910B2", - "uuid": "1E82A664 100D179 346CBD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "3281A664 1404B1E 1AC3572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3408, - "memory_utilization": 5.2, + "memory_used": 3420, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 104.5, + "power_used": 102.6, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:41:00.0", - "numa": "2", "card_id": 5, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.12", + "physical_id": 5, + "numa": "2", + "roce_ip": "10.52.32.17", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 6, "name": "910B2", - "uuid": "3EC1A664 60C27A 1902B172 99D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "7281A664 1207247 20B9E972 99D00485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3408, - "memory_utilization": 5.2, + "memory_used": 3419, + "memory_utilization": 5.22, "memory_status": "healthy", - "temperature": 51, + "temperature": 45, "power": null, - "power_used": 116.6, + "power_used": 102.3, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:82:00.0", - "numa": "4", "card_id": 6, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.10", + "physical_id": 6, + "numa": "4", + "roce_ip": "10.52.32.15", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } }, { "manufacturer": "ascend", "index": 7, "name": "910B2", - "uuid": "7281A664 E0C977 690ABD72 B9D00485 104301E3", - "driver_version": "25.2.0", - "runtime_version": null, - "runtime_version_original": null, + "uuid": "785BA664 C0D378 3411B572 86D28485 104301E3", + "driver_version": "25.5.1", + "runtime_version": "8.5", + "runtime_version_original": "8.5.0", "compute_capability": null, "cores": 24, "cores_utilization": 0, "memory": 65536, - "memory_used": 3409, - "memory_utilization": 5.2, + "memory_used": 3426, + "memory_utilization": 5.23, "memory_status": "healthy", - "temperature": 51, + "temperature": 47, "power": null, - "power_used": 102.2, + "power_used": 99.9, "appendix": { "arch_family": "Ascend910B2", - "vgpu": false, "bdf": "0000:42:00.0", - "numa": "2", "card_id": 7, "device_id": 0, "device_id_max": 0, - "roce_ip": "10.52.65.9", + "physical_id": 7, + "numa": "2", + "roce_ip": "10.52.32.14", "roce_mask": "255.255.255.0", - "roce_gateway": "10.52.65.1" + "roce_gateway": "10.52.32.1" } } ] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json b/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json index 88a606f..41545ca 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_hygon_k100ai.json @@ -18,7 +18,6 @@ "power": 400, "power_used": 104, "appendix": { - "vgpu": false, "bdf": "0000:07:00.0", "numa": "0", "card_id": 1, @@ -44,7 +43,6 @@ "power": 400, "power_used": 107, "appendix": { - "vgpu": false, "bdf": "0000:0a:00.0", "numa": "0", "card_id": 2, @@ -70,7 +68,6 @@ "power": 400, "power_used": 106, "appendix": { - "vgpu": false, "bdf": "0000:0f:00.0", "numa": "0", "card_id": 3, @@ -96,7 +93,6 @@ "power": 400, "power_used": 105, "appendix": { - "vgpu": false, "bdf": "0000:16:00.0", "numa": "0", "card_id": 4, @@ -122,7 +118,6 @@ "power": 400, "power_used": 108, "appendix": { - "vgpu": false, "bdf": "0000:1d:00.0", "numa": "0", "card_id": 5, @@ -148,7 +143,6 @@ "power": 400, "power_used": 110, "appendix": { - "vgpu": false, "bdf": "0000:20:00.0", "numa": "0", "card_id": 6, @@ -174,7 +168,6 @@ "power": 400, "power_used": 110, "appendix": { - "vgpu": false, "bdf": "0000:25:00.0", "numa": "0", "card_id": 7, @@ -200,7 +193,6 @@ "power": 400, "power_used": 108, "appendix": { - "vgpu": false, "bdf": "0000:2c:00.0", "numa": "0", "card_id": 8, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json b/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json index 8b64535..69d1b55 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_metax_c500.json @@ -18,7 +18,6 @@ "power": 350, "power_used": 71, "appendix": { - "vgpu": false, "bdf": "0000:0e:00.0", "numa": "0", "card_id": 1, @@ -44,7 +43,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:0f:00.0", "numa": "0", "card_id": 2, @@ -70,7 +68,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:10:00.0", "numa": "0", "card_id": 3, @@ -96,7 +93,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:12:00.0", "numa": "0", "card_id": 4, @@ -122,7 +118,6 @@ "power": 350, "power_used": 57, "appendix": { - "vgpu": false, "bdf": "0000:35:00.0", "numa": "0", "card_id": 5, @@ -148,7 +143,6 @@ "power": 350, "power_used": 58, "appendix": { - "vgpu": false, "bdf": "0000:36:00.0", "numa": "0", "card_id": 6, @@ -174,7 +168,6 @@ "power": 350, "power_used": 56, "appendix": { - "vgpu": false, "bdf": "0000:37:00.0", "numa": "0", "card_id": 7, @@ -200,7 +193,6 @@ "power": 350, "power_used": 56, "appendix": { - "vgpu": false, "bdf": "0000:38:00.0", "numa": "0", "card_id": 8, diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json index 89734b2..ffb2d4c 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_gb10.json @@ -19,7 +19,7 @@ "power_used": 12, "appendix": { "arch_family": "Blackwell", - "vgpu": false, + "mig": false, "bdf": "0000:ca:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json index 2d66611..aa08aa9 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100.json @@ -19,7 +19,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:0e:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -46,7 +46,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:15:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -73,7 +73,7 @@ "power_used": 67, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:1b:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -100,7 +100,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:67:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -127,7 +127,7 @@ "power_used": 69, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:9a:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -154,7 +154,7 @@ "power_used": 68, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ab:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -181,7 +181,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ba:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -208,7 +208,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:db:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json index f451245..54d3464 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h100_mig.json @@ -1,92 +1,102 @@ [ { "manufacturer": "nvidia", - "index": 8, - "name": "MIG 1g.20gb", - "uuid": "MIG-667481b3-d832-5bc3-81ee-2f27a4f6cdf2", + "index": 0, "driver_version": "575.57.08", "runtime_version": "12.9", "runtime_version_original": "12.9.0", "compute_capability": "9.0", - "cores": 26, - "cores_utilization": 0, - "memory": 20096, - "memory_used": 16, - "memory_utilization": 0.08, - "memory_status": "healthy", - "temperature": 43, - "power": 700, - "power_used": 71, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 6, - "compute_instance_id": 0, - "gpu_instance_index": 57, - "compute_instance_index": 58 - } - }, - { - "manufacturer": "nvidia", - "index": 9, - "name": "MIG 1g.10gb", - "uuid": "MIG-30be8fbd-3a0a-59e7-ab47-88215d1c7022", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 9984, - "memory_used": 16, - "memory_utilization": 0.16, - "memory_status": "healthy", - "temperature": 43, - "power": 700, - "power_used": 71, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 11, - "compute_instance_id": 0, - "gpu_instance_index": 102, - "compute_instance_index": 103 - } - }, - { - "manufacturer": "nvidia", - "index": 10, - "name": "MIG 1g.10gb", - "uuid": "MIG-2808b45e-8825-5a89-8210-4f096cb502fe", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 9984, - "memory_used": 16, - "memory_utilization": 0.16, - "memory_status": "healthy", "temperature": 43, "power": 700, "power_used": 71, "appendix": { "arch_family": "Hopper", - "vgpu": true, - "sliced": true, + "mig": true, "bdf": "0000:00:0a.0", - "numa": "0-1", - "gpu_instance_id": 12, - "compute_instance_id": 0, - "gpu_instance_index": 111, - "compute_instance_index": 112 + "mig_devices": [ + { + "index": 8, + "name": "1g.20gb", + "uuid": "MIG-667481b3-d832-5bc3-81ee-2f27a4f6cdf2", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 26, + "cores_utilization": 0, + "memory": 20096, + "memory_used": 16, + "memory_utilization": 0.08, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 6, + "compute_instance_id": 0 + } + }, + { + "index": 9, + "name": "1g.10gb", + "uuid": "MIG-30be8fbd-3a0a-59e7-ab47-88215d1c7022", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 9984, + "memory_used": 16, + "memory_utilization": 0.16, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 11, + "compute_instance_id": 0 + } + }, + { + "index": 10, + "name": "1g.10gb", + "uuid": "MIG-2808b45e-8825-5a89-8210-4f096cb502fe", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 9984, + "memory_used": 16, + "memory_utilization": 0.16, + "memory_status": "healthy", + "temperature": 43, + "power": 700, + "power_used": 71, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0a.0", + "numa": "0-1", + "gpu_instance_id": 12, + "compute_instance_id": 0 + } + } + ], + "numa": "0-1" } }, { @@ -109,7 +119,7 @@ "power_used": 73, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0b.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -136,7 +146,7 @@ "power_used": 70, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0c.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -145,62 +155,75 @@ }, { "manufacturer": "nvidia", - "index": 32, - "name": "MIG 1c.3g.40gb", - "uuid": "MIG-a72639ea-72c4-5068-adaa-30ed6ec08a86", + "index": 3, "driver_version": "575.57.08", "runtime_version": "12.9", "runtime_version_original": "12.9.0", "compute_capability": "9.0", - "cores": 16, - "cores_utilization": 0, - "memory": 40448, - "memory_used": 46, - "memory_utilization": 0.11, - "memory_status": "healthy", "temperature": 41, "power": 700, "power_used": 73, "appendix": { "arch_family": "Hopper", - "vgpu": true, - "sliced": true, + "mig": true, "bdf": "0000:00:0d.0", - "numa": "0-1", - "gpu_instance_id": 2, - "compute_instance_id": 0, - "gpu_instance_index": 426, - "compute_instance_index": 427 - } - }, - { - "manufacturer": "nvidia", - "index": 33, - "name": "MIG 1c.3g.40gb", - "uuid": "MIG-cc07225c-ed33-5047-aff0-83a245a0c73a", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", - "compute_capability": "9.0", - "cores": 26, - "cores_utilization": 0, - "memory": 40448, - "memory_used": 46, - "memory_utilization": 0.11, - "memory_status": "healthy", - "temperature": 41, - "power": 700, - "power_used": 73, - "appendix": { - "arch_family": "Hopper", - "vgpu": true, - "sliced": true, - "bdf": "0000:00:0d.0", - "numa": "0-1", - "gpu_instance_id": 2, - "compute_instance_id": 1, - "gpu_instance_index": 426, - "compute_instance_index": 428 + "mig_devices": [ + { + "index": 32, + "name": "1c.3g.40gb", + "uuid": "MIG-a72639ea-72c4-5068-adaa-30ed6ec08a86", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 16, + "cores_utilization": 0, + "memory": 40448, + "memory_used": 46, + "memory_utilization": 0.11, + "memory_status": "healthy", + "temperature": 41, + "power": 700, + "power_used": 73, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0d.0", + "numa": "0-1", + "gpu_instance_id": 2, + "compute_instance_id": 0 + } + }, + { + "index": 33, + "name": "1c.3g.40gb", + "uuid": "MIG-cc07225c-ed33-5047-aff0-83a245a0c73a", + "driver_version": "575.57.08", + "runtime_version": "12.9", + "runtime_version_original": "12.9.0", + "compute_capability": "9.0", + "cores": 26, + "cores_utilization": 0, + "memory": 40448, + "memory_used": 46, + "memory_utilization": 0.11, + "memory_status": "healthy", + "temperature": 41, + "power": 700, + "power_used": 73, + "appendix": { + "arch_family": "Hopper", + "sliced": true, + "mig": true, + "bdf": "0000:00:0d.0", + "numa": "0-1", + "gpu_instance_id": 2, + "compute_instance_id": 1 + } + } + ], + "numa": "0-1" } }, { @@ -223,7 +246,7 @@ "power_used": 74, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0e.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -250,7 +273,7 @@ "power_used": 72, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:0f.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -277,7 +300,7 @@ "power_used": 78, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:10.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -304,7 +327,7 @@ "power_used": 71, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:00:11.0", "numa": "0-1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json index e74ed42..b2c2c26 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_h200.json @@ -19,7 +19,7 @@ "power_used": 685, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:18:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -46,7 +46,7 @@ "power_used": 701, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:29:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -73,7 +73,7 @@ "power_used": 699, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:3a:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -100,7 +100,7 @@ "power_used": 689, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:4b:00.0", "numa": "0", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -127,7 +127,7 @@ "power_used": 683, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:9a:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -154,7 +154,7 @@ "power_used": 698, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:aa:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -181,7 +181,7 @@ "power_used": 693, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ba:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", @@ -208,7 +208,7 @@ "power_used": 692, "appendix": { "arch_family": "Hopper", - "vgpu": false, + "mig": false, "bdf": "0000:ca:00.0", "numa": "1", "fabric_cluster_uuid": "00000000-0000-0000-0000-000000000000", diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json index 222b95a..d0cc879 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4080super.json @@ -19,7 +19,7 @@ "power_used": 11, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:01:00.0", "numa": "0" } @@ -44,7 +44,7 @@ "power_used": 5, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:72:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json new file mode 100644 index 0000000..1c8eb5f --- /dev/null +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090_48g.json @@ -0,0 +1,54 @@ +[ + { + "manufacturer": "nvidia", + "index": 0, + "name": "NVIDIA GeForce RTX 4090", + "uuid": "GPU-bbc521eb-4a08-800b-eb2a-770a4a5b8a8c", + "driver_version": "610.43.02", + "runtime_version": "13.3", + "runtime_version_original": "13.3.0", + "compute_capability": "8.9", + "cores": 16384, + "cores_utilization": 0, + "memory": 49140, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 450, + "power_used": 29, + "appendix": { + "arch_family": "Ada-Lovelace", + "mig": false, + "bdf": "0000:02:00.0", + "minor_number": 0, + "numa": "0" + } + }, + { + "manufacturer": "nvidia", + "index": 1, + "name": "NVIDIA GeForce RTX 4090", + "uuid": "GPU-1068d7f6-26eb-5225-371e-58ca20150673", + "driver_version": "610.43.02", + "runtime_version": "13.3", + "runtime_version_original": "13.3.0", + "compute_capability": "8.9", + "cores": 16384, + "cores_utilization": 0, + "memory": 49140, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 450, + "power_used": 31, + "appendix": { + "arch_family": "Ada-Lovelace", + "mig": false, + "bdf": "0000:03:00.0", + "minor_number": 1, + "numa": "0" + } + } +] diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json index a7b5a1e..8d8f022 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx4090d.json @@ -19,7 +19,7 @@ "power_used": 10, "appendix": { "arch_family": "Ada-Lovelace", - "vgpu": false, + "mig": false, "bdf": "0000:04:00.0", "numa": "0" } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json index 101b449..9fd07c7 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_nvidia_rtx5090d.json @@ -4,23 +4,24 @@ "index": 0, "name": "NVIDIA GeForce RTX 5090 D", "uuid": "GPU-31b58f27-ac45-ec52-c386-90aaeff2dfac", - "driver_version": "575.57.08", - "runtime_version": "12.9", - "runtime_version_original": "12.9.0", + "driver_version": "595.84", + "runtime_version": "13.2", + "runtime_version_original": "13.2.0", "compute_capability": "12.0", - "cores": 170, + "cores": 21760, "cores_utilization": 0, "memory": 32607, - "memory_used": 501, - "memory_utilization": 1.54, + "memory_used": 3, + "memory_utilization": 0.01, "memory_status": "healthy", - "temperature": 46, + "temperature": 51, "power": 575, "power_used": 21, "appendix": { "arch_family": "Blackwell", - "vgpu": false, + "mig": false, "bdf": "0000:01:00.0", + "minor_number": 0, "numa": "0" } } diff --git a/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json b/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json index c2da1a9..45e84eb 100644 --- a/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json +++ b/tests/gpustack_runtime/detector/samples/detect_output_thead_ppu.json @@ -3,47 +3,399 @@ "manufacturer": "thead", "index": 0, "name": "PPU-ZW810E", - "uuid": "GPU-011ef111-8330-0426-0000-00006093bb5b", - "driver_version": "1.4.1-816bc0", - "runtime_version": "12.3", - "runtime_version_original": "12.3.0", + "uuid": "GPU-019e2226-4211-0208-0000-000000ab261d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, "compute_capability": "8.0", - "cores": 32768, + "cores": 16384, "cores_utilization": 0, - "memory": 97920, - "memory_used": 2, - "memory_utilization": 0.0, + "memory": 98304, + "memory_used": 91349, + "memory_utilization": 92.93, "memory_status": "healthy", - "temperature": 29, + "temperature": 34, "power": 400, - "power_used": 58, + "power_used": 93, "appendix": { - "vgpu": false, - "bdf": "0000:05:00.0", - "numa": "0" + "mig": false, + "bdf": "0000:a8:00.0", + "minor_number": 1, + "numa": "4" } }, { "manufacturer": "thead", "index": 1, "name": "PPU-ZW810E", - "uuid": "GPU-011ef111-8251-0426-0000-00006035ca38", - "driver_version": "1.4.1-816bc0", - "runtime_version": "12.3", - "runtime_version_original": "12.3.0", + "uuid": "GPU-019e2226-84c1-0200-0000-0000c020e701", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:a7:00.0", + "minor_number": 2, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 2, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-4422-0024-0000-000020178514", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:a9:00.0", + "minor_number": 3, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 3, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8261-0222-0000-0000a0490d60", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 31, + "power": 400, + "power_used": 94, + "appendix": { + "mig": false, + "bdf": "0000:aa:00.0", + "minor_number": 4, + "numa": "4" + } + }, + { + "manufacturer": "thead", + "index": 4, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-84b1-061e-0000-000020cc396a", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 33, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:65:00.0", + "minor_number": 5, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 5, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0571-0120-0000-0000405f5716", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:66:00.0", + "minor_number": 6, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 6, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8491-0424-0000-000020b1f22a", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 400, + "power_used": 95, + "appendix": { + "mig": false, + "bdf": "0000:64:00.0", + "minor_number": 7, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 7, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0571-031c-0000-0000e03d0464", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 37, + "power": 400, + "power_used": 97, + "appendix": { + "mig": false, + "bdf": "0000:63:00.0", + "minor_number": 8, + "numa": "2" + } + }, + { + "manufacturer": "thead", + "index": 8, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8811-0618-0000-0000408ed50b", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 93, + "appendix": { + "mig": false, + "bdf": "0000:e8:00.0", + "minor_number": 9, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 9, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8491-0228-0000-000060ce486d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 94, + "appendix": { + "mig": false, + "bdf": "0000:e7:00.0", + "minor_number": 10, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 10, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8861-0430-0000-0000201ea851", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 35, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:e9:00.0", + "minor_number": 11, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 11, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8811-0420-0000-0000a01d582e", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 32, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:ea:00.0", + "minor_number": 12, + "numa": "6" + } + }, + { + "manufacturer": "thead", + "index": 12, + "name": "PPU-ZW810E", + "uuid": "GPU-01deb21c-8851-021a-0000-000000e57d6f", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, "compute_capability": "8.0", - "cores": 32768, + "cores": 16384, "cores_utilization": 0, - "memory": 97920, - "memory_used": 2, + "memory": 98304, + "memory_used": 0, "memory_utilization": 0.0, "memory_status": "healthy", "temperature": 32, "power": 400, - "power_used": 59, + "power_used": 92, + "appendix": { + "mig": false, + "bdf": "0000:25:00.0", + "minor_number": 13, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 13, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-0581-0510-0000-00004031af7d", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 91, + "appendix": { + "mig": false, + "bdf": "0000:26:00.0", + "minor_number": 14, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 14, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8591-061e-0000-000020791642", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 62, + "appendix": { + "mig": false, + "bdf": "0000:24:00.0", + "minor_number": 15, + "numa": "0" + } + }, + { + "manufacturer": "thead", + "index": 15, + "name": "PPU-ZW810E", + "uuid": "GPU-019e2226-8591-0510-0000-0000a066de79", + "driver_version": "2.1.0-ra1f23041713", + "runtime_version": null, + "runtime_version_original": null, + "compute_capability": "8.0", + "cores": 16384, + "cores_utilization": 0, + "memory": 98304, + "memory_used": 0, + "memory_utilization": 0.0, + "memory_status": "healthy", + "temperature": 34, + "power": 400, + "power_used": 64, "appendix": { - "vgpu": false, - "bdf": "0000:0b:00.0", + "mig": false, + "bdf": "0000:23:00.0", + "minor_number": 16, "numa": "0" } } diff --git a/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json b/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json index 8b1ae4e..8515803 100644 --- a/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json +++ b/tests/gpustack_runtime/detector/samples/topology_output_amd_rx7800xt.json @@ -3,16 +3,24 @@ "manufacturer": "amd", "devices_distances": [ [ + 0, + 50 + ], + [ + 50, 0 ] ], "devices_cpu_affinities": [ - "0-19" + "0-13", + "0-13" ], "devices_numa_affinities": [ + "0", "0" ], "appendices": [ + {}, {} ] } diff --git a/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json b/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json new file mode 100644 index 0000000..990d356 --- /dev/null +++ b/tests/gpustack_runtime/detector/samples/topology_output_nvidia_rtx4090_48g.json @@ -0,0 +1,27 @@ +[ + { + "manufacturer": "nvidia", + "devices_distances": [ + [ + 0, + 30 + ], + [ + 30, + 0 + ] + ], + "devices_cpu_affinities": [ + "0-13", + "0-13" + ], + "devices_numa_affinities": [ + "0", + "0" + ], + "appendices": [ + {}, + {} + ] + } +] diff --git a/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json b/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json index e0820ca..8f5e00b 100644 --- a/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json +++ b/tests/gpustack_runtime/detector/samples/topology_output_thead_ppu.json @@ -4,31 +4,409 @@ "devices_distances": [ [ 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, 5 ], [ 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 10 + ], + [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 10, 0 ] ], "devices_cpu_affinities": [ - "0-15", - "0-15" + "64-79,192-207", + "64-79,192-207", + "64-79,192-207", + "64-79,192-207", + "32-47,160-175", + "32-47,160-175", + "32-47,160-175", + "32-47,160-175", + "96-111,224-239", + "96-111,224-239", + "96-111,224-239", + "96-111,224-239", + "0-15,128-143", + "0-15,128-143", + "0-15,128-143", + "0-15,128-143" ], "devices_numa_affinities": [ + "4", + "4", + "4", + "4", + "2", + "2", + "2", + "2", + "6", + "6", + "6", + "6", + "0", + "0", "0", "0" ], "appendices": [ { "links_count": 7, - "links_state": 12, - "links_active_count": 2 + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 + }, + { + "links_count": 7, + "links_state": 0, + "links_active_count": 0 }, { "links_count": 7, - "links_state": 12, - "links_active_count": 2 + "links_state": 0, + "links_active_count": 0 } ] } diff --git a/tests/gpustack_runtime/detector/test_amd.py b/tests/gpustack_runtime/detector/test_amd.py index 402f539..918b961 100644 --- a/tests/gpustack_runtime/detector/test_amd.py +++ b/tests/gpustack_runtime/detector/test_amd.py @@ -1,5 +1,21 @@ +from __future__ import annotations + +import contextlib +from types import SimpleNamespace + import pytest +from gpustack_runtime.deployer.cdi import amd as cdi_amd +from gpustack_runtime.deployer.cdi.__types__ import ConfigDeviceNode +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + amd, + pyamdgpu, + pyhsa, +) +from gpustack_runtime.detector.__utils__ import _load_pci_device_names from gpustack_runtime.detector.amd import AMDDetector @@ -21,3 +37,536 @@ def test_get_topology(): det = AMDDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# Fake bindings: no AMD driver exists in this suite, so every behaviour below # +# is proved against stand-ins for pyamdsmi / pyrocmsmi / pyhsa, each recording # +# the calls it serves -- "detect_info asks for no usage" is a statement about # +# the calls made, not about the values returned. # +# --------------------------------------------------------------------------- # + + +class _FakeAmdSmiError(Exception): + """ + The fake AMD SMI's own error type. + + The detector catches whatever the binding module it is handed exposes, so + the fake brings its own error rather than constructing the real amdsmi + package's, which takes a status code and is absent here anyway. + """ + + +class _FakeRocmSmiError(Exception): + """ + The fake ROCm SMI's own error type. + """ + + +class _FakeAmdSmi: + """ + A pyamdsmi stand-in whose processor handles are the fixture cards + themselves, so every per-device answer is read out of the card. + """ + + AmdSmiException = _FakeAmdSmiError + + class AmdSmiGpuBlock: + UMC = 1 + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def amdsmi_init(self, *_args): + self.calls.append("amdsmi_init") + + def amdsmi_get_rocm_version(self) -> str: + return "6.4.1-123" + + def amdsmi_get_processor_handles(self) -> list[dict]: + return self.cards + + def amdsmi_get_gpu_asic_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_asic_info") + return dev["asic_info"] + + def amdsmi_get_gpu_device_bdf(self, dev: dict) -> str: + return dev["bdf"] + + def amdsmi_get_gpu_driver_info(self, dev: dict) -> dict: + return dev["driver_info"] + + def amdsmi_get_gpu_vram_usage(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_vram_usage") + return dev["vram"] + + def amdsmi_get_gpu_ecc_count(self, dev: dict, _block: int) -> dict: + self.calls.append("amdsmi_get_gpu_ecc_count") + return dev["ecc"] + + def amdsmi_get_power_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_power_info") + return dev["power"] + + def amdsmi_get_gpu_metrics_info(self, dev: dict) -> dict: + self.calls.append("amdsmi_get_gpu_metrics_info") + if dev["metrics"] is None: + msg = "GPU metrics are not supported" + raise _FakeAmdSmiError(msg) + return dev["metrics"] + + def amdsmi_topo_get_numa_node_number(self, dev: dict) -> int: + return dev["numa"] + + def amdsmi_get_xgmi_info(self, dev: dict) -> dict: + return dev["xgmi"] + + +class _FakeRocmSmi: + """ + A pyrocmsmi stand-in serving the fallbacks the AMD path keeps for a driver + whose AMD SMI answers nothing. + """ + + ROCMSMIError = _FakeRocmSmiError + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def rsmi_init(self, *_args): + self.calls.append("rsmi_init") + + def rsmi_dev_busy_percent_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_busy_percent_get") + return self.cards[dev_idx]["busy_percent"] + + def rsmi_dev_temp_metric_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_temp_metric_get") + return self.cards[dev_idx]["temperature"] + + def rsmi_dev_power_cap_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_cap_get") + return self.cards[dev_idx]["power_cap"] + + def rsmi_dev_power_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_get") + return self.cards[dev_idx]["power_used"] + + +class _FakeHSA: + """ + A pyhsa stand-in returning the fixture agents. + """ + + Agent = pyhsa.Agent + + def __init__(self, agents: list): + self.agents = agents + + def get_agents(self) -> list: + return self.agents + + +# An extract of the real pci.ids: the board's name lives on the subsystem line, +# which is what the operator's GetName prefers. +_PCI_IDS = """\ +1002 Advanced Micro Devices, Inc. [AMD/ATI] +\t74a1 Aqua Vanjaram +\t\t1002 0e3b Instinct MI300X OAM +""" + +_PCI_IDS_NAME = "Instinct MI300X OAM" +_HSA_NAME = "AMD Instinct MI300X" +_ASIC_MARKET_NAME = "Aqua Vanjaram" +_MARKETING_NAME = "AMD Instinct MI300X OAM" + + +class _FakeAmdGpu: + """ + A libdrm stand-in exposing only what the name precedence reaches for. + + The real call answers with a pointer into libdrm's own table, and NULL for + a device id the table does not carry, which the binding reports as "". + """ + + def __init__(self, marketing_name: str): + self._marketing_name = marketing_name + + def __getattr__(self, name): + # Everything not faked here -- the error type and the family constants + # -- comes from the real binding, so the fake cannot drift from it. + return getattr(pyamdgpu, name) + + @contextlib.contextmanager + def amdgpu_device(self, card): + yield SimpleNamespace(card=card) + + def amdgpu_get_marketing_name(self, _device) -> str: + return self._marketing_name + + def amdgpu_query_gpu_info(self, _device): + return SimpleNamespace(cu_active_number=0, family_id=0) + + +_MEMORY_TOTAL = 196592 + + +def _card( + bdf: str, + serial: str, + *, + gfx_activity: int = 37, + hotspot: int = 58, + vram_used: int = 1024, + socket_power: int = 142, + no_metrics: bool = False, +) -> dict: + """ + One fake AMD card: the dict is the processor handle the fake AMD SMI hands + out, and the PCI IDs the fixture sysfs tree exposes for its BDF. + """ + return { + "bdf": bdf, + "pci_ids": { + "vendor": "0x1002", + "device": "0x74a1", + "subsystem_vendor": "0x1002", + "subsystem_device": "0x0e3b", + }, + "asic_info": { + "market_name": _ASIC_MARKET_NAME, + "asic_serial": serial, + "target_graphics_version": "gfx942", + }, + "driver_info": {"driver_version": "6.12.12"}, + "vram": {"vram_total": _MEMORY_TOTAL, "vram_used": vram_used}, + "ecc": {"uncorrectable_count": 0}, + "power": { + "power_limit": 750000000, + "current_socket_power": socket_power, + "average_socket_power": 138, + }, + "metrics": None + if no_metrics + else { + "average_gfx_activity": gfx_activity, + "temperature_hotspot": hotspot, + }, + "busy_percent": 23, + "temperature": 47, + "power_cap": 700, + "power_used": 131, + "numa": 0, + "xgmi": {"xgmi_lanes": 16, "xgmi_hive_id": "0x2b", "xgmi_node_id": 3}, + } + + +def _agent(bdf: str) -> pyhsa.Agent: + return pyhsa.Agent( + device_type=1, + device_id="0x74a1", + bdf=bdf, + uuid="", + name=_HSA_NAME, + compute_capability="gfx942", + compute_units=304, + ) + + +@pytest.fixture +def amd_bindings(monkeypatch, tmp_path): + """ + Drive the AMD detector off the fake bindings, a fixture sysfs PCI tree and + a fixture pci.ids database, returning the shared call log. + """ + + def _setup( + cards: list[dict], + agents: list | None = None, + pci_ids: str | None = _PCI_IDS, + ) -> list[str]: + calls: list[str] = [] + + monkeypatch.setattr(AMDDetector, "is_supported", staticmethod(lambda: True)) + monkeypatch.setattr(amd, "pyamdsmi", _FakeAmdSmi(calls, cards)) + monkeypatch.setattr(amd, "pyrocmsmi", _FakeRocmSmi(calls, cards)) + monkeypatch.setattr(amd, "pyhsa", _FakeHSA(list(agents or []))) + + pci_devices_path = tmp_path / "pci_devices" + for card in cards: + card_path = pci_devices_path / card["bdf"] + card_path.mkdir(parents=True) + for name, value in card["pci_ids"].items(): + (card_path / name).write_text(f"{value}\n") + monkeypatch.setattr(amd, "_PCI_DEVICES_PATH", pci_devices_path) + + pci_ids_paths: tuple[str, ...] = () + if pci_ids is not None: + pci_ids_path = tmp_path / "pci.ids" + pci_ids_path.write_text(pci_ids, encoding="utf-8") + pci_ids_paths = (str(pci_ids_path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + pci_ids_paths, + ) + _load_pci_device_names.cache_clear() + + return calls + + yield _setup + + _load_pci_device_names.cache_clear() + + +# --------------------------------------------------------------------------- # +# detect_info: the device name's precedence, the inventory, and the usage # +# calls it must not make. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_pci_ids_name(amd_bindings): + # The operator resolves the board's name from pci.ids before asking the + # driver, which only knows the chip. + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_PCI_IDS_NAME] + + +def test_detect_info_falls_back_to_the_hsa_product_name(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + pci_ids=None, + ) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_HSA_NAME] + + +def test_detect_info_falls_back_to_the_asic_market_name(amd_bindings): + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_ASIC_MARKET_NAME] + + +def test_detect_info_asks_libdrm_for_the_marketing_name(amd_bindings, monkeypatch): + # Between the HSA name and the ASIC name the operator asks libdrm for the + # board's marketing name. With pci.ids and HSA both silent, that answer is + # the one reported -- the ASIC market name is only the step after it. + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + monkeypatch.setattr(amd, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr(amd, "pyamdgpu", _FakeAmdGpu(marketing_name=_MARKETING_NAME)) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_MARKETING_NAME] + + +def test_detect_info_falls_through_an_unnamed_board(amd_bindings, monkeypatch): + # libdrm's table carries no entry for every device id and answers NULL + # rather than failing, which the binding reports as an empty name. + amd_bindings([_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], pci_ids=None) + monkeypatch.setattr(amd, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr(amd, "pyamdgpu", _FakeAmdGpu(marketing_name="")) + + devices = AMDDetector().detect_info() + + assert [dev.name for dev in devices] == [_ASIC_MARKET_NAME] + + +def test_detect_info_reports_the_inventory(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00A1B2C3D4E5F600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + dev = devices[0] + assert dev.manufacturer is ManufacturerEnum.AMD + assert dev.index == 0 + # The ASIC serial, lowercased and stripped of its "0x". + assert dev.uuid == "GPU-00a1b2c3d4e5f600" + assert dev.driver_version == "6.12.12" + assert dev.runtime_version == "6.4" + assert dev.runtime_version_original == "6.4.1-123" + assert dev.compute_capability == "gfx942" + assert dev.cores == 304 + assert dev.memory == _MEMORY_TOTAL + assert dev.memory_status is DeviceMemoryStatusEnum.HEALTHY + # The power limit is inventory, not usage. + assert dev.power == 750 + assert dev.appendix["bdf"] == "0000:05:00.0" + assert dev.appendix["numa"] == "0" + assert dev.appendix["xgmi_lanes"] == 16 + + +def test_detect_reports_no_vgpu(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + det = AMDDetector() + + # There is no virtual/PF/VF classification any more, in either query. + assert "vgpu" not in det.detect_info()[0].appendix + assert "vgpu" not in det.detect()[0].appendix + + +def test_detect_info_issues_no_usage_call(amd_bindings): + calls = amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_info() + + assert "amdsmi_get_gpu_metrics_info" not in calls + assert "rsmi_dev_busy_percent_get" not in calls + assert "rsmi_dev_temp_metric_get" not in calls + assert "rsmi_dev_power_get" not in calls + # The power query stays, because the limit it also carries is inventory. + assert "amdsmi_get_power_info" in calls + + dev = devices[0] + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +# --------------------------------------------------------------------------- # +# detect_usage. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(amd_bindings): + amd_bindings( + [ + _card("0000:05:00.0", "0x00a1b2c3d4e5f600"), + _card( + "0000:06:00.0", + "0x00a1b2c3d4e5f601", + gfx_activity=91, + hotspot=72, + vram_used=8192, + socket_power=311, + ), + ], + agents=[_agent("0000:05:00.0"), _agent("0000:06:00.0")], + ) + det = AMDDetector() + devices = det.detect_info() + + # Reversed, to prove the merge joins by UUID rather than by position. + det.detect_usage(list(reversed(devices))) + + assert [dev.cores_utilization for dev in devices] == [37, 91] + assert [dev.temperature for dev in devices] == [58, 72] + assert [dev.memory_used for dev in devices] == [1024, 8192] + assert [dev.power_used for dev in devices] == [142, 311] + assert [dev.memory_utilization for dev in devices] == [0.52, 4.17] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + DeviceMemoryStatusEnum.HEALTHY, + ] + # The information fields survive the merge. + assert [dev.memory for dev in devices] == [_MEMORY_TOTAL, _MEMORY_TOTAL] + assert [dev.power for dev in devices] == [750, 750] + + +def test_detect_usage_falls_back_to_rocm_smi(amd_bindings): + calls = amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600", no_metrics=True)], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect() + + assert "rsmi_dev_busy_percent_get" in calls + assert "rsmi_dev_temp_metric_get" in calls + assert devices[0].cores_utilization == 23 + assert devices[0].temperature == 47 + + +def test_detect_usage_detects_the_information_first(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect_usage() + + assert devices[0].name == _PCI_IDS_NAME + assert devices[0].cores_utilization == 37 + assert devices[0].power_used == 142 + + +def test_detect_composes_the_information_and_the_usage(amd_bindings): + amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + devices = AMDDetector().detect() + + assert devices[0].memory == _MEMORY_TOTAL + assert devices[0].memory_used == 1024 + assert devices[0].power == 750 + assert devices[0].power_used == 142 + + +# --------------------------------------------------------------------------- # +# The CDI generator, which numbers its device nodes from the appendix. # +# --------------------------------------------------------------------------- # + + +def _fake_device_node(path: str, **_kwargs) -> ConfigDeviceNode: + """ + Stand in for the host's device nodes: neither /dev/kfd nor /dev/dri exists + in this suite, so the real lookup would drop every path. + """ + return ConfigDeviceNode(path=path, type_="c") + + +def test_cdi_spec_numbers_the_device_nodes_from_the_appendix(monkeypatch): + monkeypatch.setattr(cdi_amd, "device_to_cdi_device_node", _fake_device_node) + + config = cdi_amd.AMDGenerator().generate( + [ + Device( + manufacturer=ManufacturerEnum.AMD, + index=0, + name=_PCI_IDS_NAME, + uuid="GPU-a1b2c3d4e5f600", + appendix={"card_id": 5, "renderd_id": 133}, + ), + ], + ) + + # The DRM numbering comes from the appendix, never from Device.index -- + # here the card is enumerated at 0 and its nodes are card5 / renderD133. + device_nodes = config["devices"][0]["containerEdits"]["deviceNodes"] + assert [node["path"] for node in device_nodes] == [ + "/dev/dri/card5", + "/dev/dri/renderD133", + ] + assert [dev["name"] for dev in config["devices"]] == [ + "0", + "GPU-a1b2c3d4e5f600", + "all", + ] + assert [node["path"] for node in config["containerEdits"]["deviceNodes"]] == [ + "/dev/kfd", + ] diff --git a/tests/gpustack_runtime/detector/test_ascend.py b/tests/gpustack_runtime/detector/test_ascend.py index 461210b..1851f7a 100644 --- a/tests/gpustack_runtime/detector/test_ascend.py +++ b/tests/gpustack_runtime/detector/test_ascend.py @@ -1,5 +1,21 @@ +from __future__ import annotations + +import ctypes +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import ascend as cdi_ascend +from gpustack_runtime.deployer.cdi.ascend import AscendGenerator +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + ascend, + pydcmi, +) +from gpustack_runtime.detector.__utils__ import get_utilization from gpustack_runtime.detector.ascend import AscendDetector @@ -21,3 +37,699 @@ def test_get_topology(): det = AscendDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# ABI of the V1 structs, pinned against dcmi_interface_api.h. # +# # +# No Ascend driver exists in CI, so nothing else would catch a mislaid field: # +# the numbers below are read off the header's declarations, not off the # +# ctypes code they check. # +# --------------------------------------------------------------------------- # + + +def test_pcie_info_v1_struct_matches_the_header(): + # struct dcmi_pcie_info { unsigned int deviceid, venderid, subvenderid, + # subdeviceid, bdf_deviceid, bdf_busid, bdf_funcid; } + assert [f[0] for f in pydcmi.c_dcmi_pcie_info._fields_] == [ + "deviceid", + "venderid", + "subvenderid", + "subdeviceid", + "bdf_deviceid", + "bdf_busid", + "bdf_funcid", + ] + assert ctypes.sizeof(pydcmi.c_dcmi_pcie_info) == 7 * 4 + for offset, name in enumerate( + [f[0] for f in pydcmi.c_dcmi_pcie_info._fields_], + ): + assert getattr(pydcmi.c_dcmi_pcie_info, name).offset == offset * 4 + # The V1 struct carries no PCI domain, where the V2 one does. The asymmetry + # is why the detector formats a domain of 0 on the fallback path. + assert not hasattr(pydcmi.c_dcmi_pcie_info, "domain") + assert hasattr(pydcmi.c_dcmi_pcie_info_all, "domain") + + +def test_memory_info_v2_struct_matches_the_header(): + # struct dcmi_memory_info { unsigned long long memory_size; unsigned int + # freq; unsigned int utiliza; } + assert [f[0] for f in pydcmi.c_dcmi_memory_info._fields_] == [ + "memory_size", + "freq", + "utiliza", + ] + assert pydcmi.c_dcmi_memory_info.memory_size.offset == 0 + assert pydcmi.c_dcmi_memory_info.memory_size.size == 8 + assert pydcmi.c_dcmi_memory_info.freq.offset == 8 + assert pydcmi.c_dcmi_memory_info.utiliza.offset == 12 + assert ctypes.sizeof(pydcmi.c_dcmi_memory_info) == 16 + # Unlike V3, it reports no available memory at all, which is why the used + # memory has to be derived from the utilization percentage. + assert not hasattr(pydcmi.c_dcmi_memory_info, "memory_available") + + +def test_die_id_struct_matches_the_header(): + # The V1 call fills a struct dcmi_soc_die_stru { unsigned int soc_die[5]; }, + # laid out exactly as the dcmi_die_id the V2 call fills. + assert ctypes.sizeof(pydcmi.c_dcmi_die_id) == 5 * 4 + assert pydcmi.c_dcmi_die_id.soc_die.offset == 0 + + +# --------------------------------------------------------------------------- # +# A fake pydcmi carrying a call log, so the type filter, the V1 fallbacks and # +# the information/usage split are provable on a host with no DCMI driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Unit: + """ + One device inside a dcmi card, as the driver would report it. + + The per-generation values are deliberately distinct, so a test can tell + which call a returned value came from. + """ + + card_id: int = 0 + device_id: int = 0 + unit_type: int = pydcmi.DCMI_UNIT_TYPE_NPU + unit_type_readable: bool = True + logic_id: int = 0 + physical_id: int = 0 + physical_id_readable: bool = True + aicore_cnt: int = 20 + ecc_errors: int = 0 + cores_utilization: int = 42 + temperature: int = 55 + power_deciwatts: int = 1234 + # Which generation of calls the driver exposes. + v2_die: bool = True + v2_pcie: bool = True + v2_chip_info: bool = True + hbm: bool = True + v3_memory: bool = True + + @property + def v2_die_id(self) -> str: + return f"1a 2b 3c 4d {self.logic_id:x}" + + @property + def v1_die_id(self) -> str: + return f"9f 8e 7d 6c {self.logic_id:x}" + + @property + def v2_chip_name(self) -> str: + return "910B3" + + @property + def v1_chip_name(self) -> str: + return "910A" + + @property + def v2_bdf(self) -> str: + return f"0001:{0x10 + self.logic_id:02x}:00.0" + + @property + def v1_bdf(self) -> str: + # The V1 struct has no domain, so the detector formats it as zero. + return f"0000:{0x20 + self.logic_id:02x}:00.0" + + +class _FakeStruct: + """ + A stand-in for a ctypes struct the binding would return. + """ + + def __init__(self, **fields): + self.__dict__.update(fields) + + +@dataclass +class _FakeDCMI: + """ + A stand-in for the pydcmi binding, recording every call the detector makes. + + The error type and the enumeration constants are the real module's, so a + fake drifting from the binding's contract fails here rather than on + hardware. Entry points are dispatched by name, as test_metax.py's fake + does, which keeps the driver's own naming out of the handlers' names. + + The vdev (vNPU) constants are deliberately *not* exposed: a reintroduced + vNPU branch fails here with an AttributeError rather than passing silently. + """ + + units: list[_Unit] = field(default_factory=lambda: [_Unit()]) + calls: list[str] = field(default_factory=list) + + DCMIError = pydcmi.DCMIError + DCMI_UNIT_TYPE_NPU = pydcmi.DCMI_UNIT_TYPE_NPU + DCMI_UNIT_TYPE_MCU = pydcmi.DCMI_UNIT_TYPE_MCU + DCMI_DIE_TYPE_VDIE = pydcmi.DCMI_DIE_TYPE_VDIE + DCMI_DEVICE_TYPE_HBM = pydcmi.DCMI_DEVICE_TYPE_HBM + DCMI_DEVICE_TYPE_DDR = pydcmi.DCMI_DEVICE_TYPE_DDR + DCMI_INPUT_TYPE_AICORE = pydcmi.DCMI_INPUT_TYPE_AICORE + DCMI_PORT_TYPE_ROCE_PORT = pydcmi.DCMI_PORT_TYPE_ROCE_PORT + DCMI_ERROR_FUNCTION_NOT_FOUND = pydcmi.DCMI_ERROR_FUNCTION_NOT_FOUND + DCMI_ERROR_NOT_SUPPORT = pydcmi.DCMI_ERROR_NOT_SUPPORT + DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER = pydcmi.DCMI_ERROR_NOT_SUPPORT_IN_CONTAINER + + # Memory, in MiB, per generation of the memory calls. + hbm_size: int = 65536 + hbm_usage: int = 1024 + v3_size: int = 32768 + v3_available: int = 24576 + v2_size: int = 16384 + v2_utiliza: int = 25 + + def __getattr__(self, name: str): + handler = { + "dcmi_init": self._init, + "dcmi_get_driver_version": self._get_driver_version, + "dcmi_get_card_list": self._get_card_list, + "dcmi_get_device_num_in_card": self._get_device_num_in_card, + "dcmi_get_device_type": self._get_device_type, + "dcmi_get_device_die_v2": self._get_device_die_v2, + "dcmi_get_device_die": self._get_device_die, + "dcmi_get_device_chip_info_v2": self._get_device_chip_info_v2, + "dcmi_get_device_chip_info": self._get_device_chip_info, + "dcmi_get_device_hbm_info": self._get_device_hbm_info, + "dcmi_get_device_memory_info_v3": self._get_device_memory_info_v3, + "dcmi_get_device_memory_info_v2": self._get_device_memory_info_v2, + "dcmi_get_device_ecc_info": self._get_device_ecc_info, + "dcmi_get_device_logic_id": self._get_device_logic_id, + "dcmi_get_device_phyid_from_logicid": self._get_phyid_from_logicid, + "dcmi_get_device_bdf": self._get_device_bdf, + "dcmi_get_device_pcie_info": self._get_device_pcie_info, + "dcmi_get_device_ip": self._get_device_ip, + "dcmi_get_affinity_cpu_info_by_device_id": self._get_affinity_cpu_info, + "dcmi_get_device_utilization_rate": self._get_device_utilization_rate, + "dcmi_get_device_temperature": self._get_device_temperature, + "dcmi_get_device_power_info": self._get_device_power_info, + }.get(name) + if handler is None: + msg = f"module pydcmi has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _unit(self, card_id: int, device_id: int) -> _Unit: + for unit in self.units: + if unit.card_id == card_id and unit.device_id == device_id: + return unit + msg = f"no such device: card {card_id}, device {device_id}" + raise AssertionError(msg) + + def _unsupported(self) -> DCMIError: + return self.DCMIError(self.DCMI_ERROR_NOT_SUPPORT) + + def _init(self) -> None: + pass + + def _get_driver_version(self) -> str: + return "24.1.0" + + def _get_card_list(self) -> tuple[int, list[int]]: + cards = sorted({unit.card_id for unit in self.units}) + return len(cards), cards + + def _get_device_num_in_card(self, card_id: int) -> int: + return len([unit for unit in self.units if unit.card_id == card_id]) + + def _get_device_type(self, card_id: int, device_id: int) -> int: + unit = self._unit(card_id, device_id) + if not unit.unit_type_readable: + raise self._unsupported() + return unit.unit_type + + def _get_device_die_v2(self, card_id: int, device_id: int, input_type: int) -> str: + assert input_type == pydcmi.DCMI_DIE_TYPE_VDIE + unit = self._unit(card_id, device_id) + if not unit.v2_die: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.v2_die_id + + def _get_device_die(self, card_id: int, device_id: int) -> str: + return self._unit(card_id, device_id).v1_die_id + + def _get_device_chip_info_v2(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.v2_chip_info: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return _FakeStruct(chip_name=unit.v2_chip_name, aicore_cnt=unit.aicore_cnt) + + def _get_device_chip_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct(chip_name=unit.v1_chip_name, aicore_cnt=unit.aicore_cnt) + + def _get_device_hbm_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.hbm: + raise self._unsupported() + return _FakeStruct(memory_size=self.hbm_size, memory_usage=self.hbm_usage) + + def _get_device_memory_info_v3(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + if not unit.v3_memory: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return _FakeStruct( + memory_size=self.v3_size, + memory_available=self.v3_available, + ) + + def _get_device_memory_info_v2(self, card_id: int, device_id: int) -> _FakeStruct: + return _FakeStruct(memory_size=self.v2_size, utiliza=self.v2_utiliza) + + def _get_device_ecc_info( + self, + card_id: int, + device_id: int, + device_type: int, + ) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct( + enable_flag=1, + single_bit_error_cnt=unit.ecc_errors, + double_bit_error_cnt=0, + ) + + def _get_device_logic_id(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).logic_id + + def _get_phyid_from_logicid(self, logic_id: int) -> int: + for unit in self.units: + if unit.logic_id == logic_id: + if not unit.physical_id_readable: + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.physical_id + msg = f"no such logic id: {logic_id}" + raise AssertionError(msg) + + def _get_device_bdf(self, card_id: int, device_id: int) -> str: + unit = self._unit(card_id, device_id) + if not unit.v2_pcie: + # As the real wrapper does: it is built on the V2 call. + raise self.DCMIError(self.DCMI_ERROR_FUNCTION_NOT_FOUND) + return unit.v2_bdf + + def _get_device_pcie_info(self, card_id: int, device_id: int) -> _FakeStruct: + unit = self._unit(card_id, device_id) + return _FakeStruct( + bdf_busid=0x20 + unit.logic_id, + bdf_deviceid=0, + bdf_funcid=0, + ) + + def _get_device_ip(self, card_id: int, device_id: int, port_type: int) -> None: + raise self._unsupported() + + def _get_affinity_cpu_info(self, card_id: int, device_id: int) -> None: + raise self._unsupported() + + def _get_device_utilization_rate( + self, + card_id: int, + device_id: int, + input_type: int, + ) -> int: + assert input_type == pydcmi.DCMI_INPUT_TYPE_AICORE + return self._unit(card_id, device_id).cores_utilization + + def _get_device_temperature(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).temperature + + def _get_device_power_info(self, card_id: int, device_id: int) -> int: + return self._unit(card_id, device_id).power_deciwatts + + +_USAGE_ONLY_CALLS = { + "dcmi_get_device_utilization_rate", + "dcmi_get_device_temperature", + "dcmi_get_device_power_info", +} + + +@pytest.fixture(autouse=True) +def _reset_is_supported_cache(): + # is_supported()/detect_pci_devices() are lru_cache'd, so a value + # observed by one test would otherwise leak into the next. + AscendDetector.is_supported.cache_clear() + AscendDetector.detect_pci_devices.cache_clear() + yield + AscendDetector.is_supported.cache_clear() + AscendDetector.detect_pci_devices.cache_clear() + + +@pytest.fixture +def fake_pydcmi(monkeypatch): + def _install(units: list[_Unit] | None = None, **kwargs) -> _FakeDCMI: + fake = _FakeDCMI(units=units if units is not None else [_Unit()], **kwargs) + monkeypatch.setattr(ascend, "pydcmi", fake) + # No PCI sysfs tree exists on the dev machine, so bypass the PCI + # presence check that is_supported() otherwise gates on. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: the NPU type filter. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_skips_a_non_npu_device(fake_pydcmi): + fake = fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=0, + device_id=1, + logic_id=1, + physical_id=1, + unit_type=pydcmi.DCMI_UNIT_TYPE_MCU, + ), + ], + ) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + # The skipped unit is never queried any further. + assert fake.calls.count("dcmi_get_device_die_v2") == 1 + + +def test_detect_info_keeps_a_device_whose_type_is_unreadable(fake_pydcmi): + # The operator skips a device only when the type call *succeeds* and + # reports something other than an NPU. + fake_pydcmi([_Unit(unit_type_readable=False)]) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + + +# --------------------------------------------------------------------------- # +# detect_info: the V1 fallbacks, one test each, on value and call log. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_v2_calls(fake_pydcmi): + fake = fake_pydcmi() + unit = fake.units[0] + + devices = AscendDetector().detect_info() + + assert devices[0].uuid == unit.v2_die_id.upper() + assert devices[0].name == unit.v2_chip_name + assert devices[0].appendix["bdf"] == unit.v2_bdf + assert devices[0].memory == fake.hbm_size + for v1_call in ( + "dcmi_get_device_die", + "dcmi_get_device_chip_info", + "dcmi_get_device_pcie_info", + "dcmi_get_device_memory_info_v3", + "dcmi_get_device_memory_info_v2", + ): + assert v1_call not in fake.calls + + +def test_detect_info_falls_back_to_the_v1_die(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_die=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].uuid == fake.units[0].v1_die_id.upper() + assert "dcmi_get_device_die_v2" in fake.calls + assert "dcmi_get_device_die" in fake.calls + + +def test_detect_info_falls_back_to_the_v1_pcie_info(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_pcie=False)]) + + devices = AscendDetector().detect_info() + + # The V1 struct carries no domain, so it reads as 0000. + assert devices[0].appendix["bdf"] == fake.units[0].v1_bdf + assert "dcmi_get_device_bdf" in fake.calls + assert "dcmi_get_device_pcie_info" in fake.calls + + +def test_detect_info_falls_back_to_the_v1_chip_info(fake_pydcmi): + fake = fake_pydcmi([_Unit(v2_chip_info=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].name == fake.units[0].v1_chip_name + assert devices[0].cores == fake.units[0].aicore_cnt + assert "dcmi_get_device_chip_info_v2" in fake.calls + assert "dcmi_get_device_chip_info" in fake.calls + + +def test_detect_info_falls_back_to_the_v3_memory_when_hbm_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].memory == fake.v3_size + assert "dcmi_get_device_memory_info_v3" in fake.calls + assert "dcmi_get_device_memory_info_v2" not in fake.calls + + +def test_detect_info_falls_back_to_the_v2_memory_when_v3_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False, v3_memory=False)]) + + devices = AscendDetector().detect_info() + + assert devices[0].memory == fake.v2_size + assert "dcmi_get_device_memory_info_v2" in fake.calls + + +def test_detect_info_yields_a_device_on_a_v1_only_driver(fake_pydcmi): + # The whole point of the fallbacks: a driver exposing none of the newer + # calls still detects. + fake = fake_pydcmi( + [ + _Unit( + v2_die=False, + v2_pcie=False, + v2_chip_info=False, + hbm=False, + v3_memory=False, + ), + ], + ) + unit = fake.units[0] + + devices = AscendDetector().detect_info() + + assert len(devices) == 1 + assert devices[0].uuid == unit.v1_die_id.upper() + assert devices[0].name == unit.v1_chip_name + assert devices[0].appendix["bdf"] == unit.v1_bdf + assert devices[0].memory == fake.v2_size + + +# --------------------------------------------------------------------------- # +# detect_info: no vGPU, no usage calls. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_has_no_vgpu_in_appendix(fake_pydcmi): + fake_pydcmi() + + devices = AscendDetector().detect_info() + + assert all("vgpu" not in dev.appendix for dev in devices) + + +def test_detect_info_keeps_the_physical_id_appendix(fake_pydcmi): + fake_pydcmi([_Unit(logic_id=1, physical_id=6)]) + + devices = AscendDetector().detect_info() + + assert devices[0].index == 1 + assert devices[0].appendix["physical_id"] == 6 + + +def test_detect_info_skips_a_device_without_a_readable_physical_id(fake_pydcmi): + # /dev/davinciN is numbered by the physical id, so a device whose physical + # id the driver will not answer cannot be addressed. Reporting it would + # offer an NPU that no CDI spec can reference, and standing the logic id in + # for the physical id would reference another NPU, so it is dropped -- as + # the operator's ascend/device.go does on a failed GetPhysicalID. + fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=1, + device_id=0, + logic_id=1, + physical_id=7, + physical_id_readable=False, + ), + ], + ) + + devices = AscendDetector().detect_info() + + assert [dev.index for dev in devices] == [0] + + +def test_detect_info_issues_no_usage_calls(fake_pydcmi): + fake = fake_pydcmi() + + AscendDetector().detect_info() + + assert _USAGE_ONLY_CALLS.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# detect_usage: merged by uuid, memory_status recomputed. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_the_usage_fields_by_uuid(fake_pydcmi): + fake = fake_pydcmi() + unit = fake.units[0] + + devices = AscendDetector().detect_info() + result = AscendDetector().detect_usage(devices) + + assert result is devices + assert devices[0].cores_utilization == unit.cores_utilization + assert devices[0].memory_used == fake.hbm_usage + assert devices[0].memory_utilization == get_utilization( + fake.hbm_usage, + fake.hbm_size, + ) + assert devices[0].temperature == unit.temperature + assert devices[0].power_used == unit.power_deciwatts / 10 + + +def test_detect_usage_recomputes_the_memory_status(fake_pydcmi, monkeypatch): + # merge_devices_usage overwrites memory_status, so the usage pass has to + # produce it as well or the inventory pass's verdict is lost. + fake_pydcmi([_Unit(ecc_errors=3)]) + # The ECC read costs a driver call per device, so the health check is off + # by default. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + + devices = AscendDetector().detect_info() + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + AscendDetector().detect_usage(devices) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_usage_keeps_a_healthy_status_healthy(fake_pydcmi): + fake_pydcmi() + + devices = AscendDetector().detect_usage(AscendDetector().detect_info()) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_usage_skips_a_non_npu_device(fake_pydcmi): + fake = fake_pydcmi( + [ + _Unit(card_id=0, device_id=0, logic_id=0, physical_id=0), + _Unit( + card_id=0, + device_id=1, + logic_id=1, + physical_id=1, + unit_type=pydcmi.DCMI_UNIT_TYPE_MCU, + ), + ], + ) + + devices = AscendDetector().detect_info() + fake.calls.clear() + AscendDetector().detect_usage(devices) + + # The filter applies to the usage pass too, as it does in the operator's + # MonitorAccelerator. + assert fake.calls.count("dcmi_get_device_utilization_rate") == 1 + + +def test_detect_usage_falls_back_to_the_v2_memory_when_v3_unavailable(fake_pydcmi): + fake = fake_pydcmi([_Unit(hbm=False, v3_memory=False)]) + + devices = AscendDetector().detect_usage(AscendDetector().detect_info()) + + # The V2 struct reports no available memory, so the used memory comes from + # its utilization percentage -- the operator instead subtracts an + # always-zero available figure and so reports the card as fully used. + assert devices[0].memory_used == fake.v2_size * fake.v2_utiliza // 100 + assert devices[0].memory_used != fake.v2_size + + +def test_detect_composes_info_and_usage_by_default(fake_pydcmi): + fake = fake_pydcmi() + + devices = AscendDetector().detect() + + assert devices[0].cores_utilization == fake.units[0].cores_utilization + assert devices[0].power_used == fake.units[0].power_deciwatts / 10 + assert "vgpu" not in devices[0].appendix + + +def test_detect_without_usage_issues_no_usage_calls(fake_pydcmi): + fake = fake_pydcmi() + + devices = AscendDetector().detect(usage=False) + + assert len(devices) == 1 + assert _USAGE_ONLY_CALLS.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# CDI: /dev/davinci{N} only, from the appendix physical id. # +# --------------------------------------------------------------------------- # + + +def test_cdi_emits_no_vdavinci_path(monkeypatch): + seen_paths: list[str] = [] + + def _fake_device_node(path, **_kwargs): + seen_paths.append(path) + return {"path": path} + + monkeypatch.setattr(cdi_ascend, "device_to_cdi_device_node", _fake_device_node) + monkeypatch.setattr(cdi_ascend, "path_to_cdi_mount", lambda **_kwargs: None) + + devices = [ + # A stale vgpu appendix must not resurrect the /dev/vdavinci path. + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=0, + name="910B3", + uuid="DIE-0", + memory=65536, + appendix={"card_id": 0, "device_id": 0, "physical_id": 3, "vgpu": True}, + ), + Device( + manufacturer=ManufacturerEnum.ASCEND, + index=1, + name="910B3", + uuid="DIE-1", + memory=65536, + appendix={"card_id": 1, "device_id": 0}, + ), + ] + + config = AscendGenerator().generate(devices) + + assert config is not None + assert not any("vdavinci" in path for path in seen_paths) + # The device carrying a physical id is addressed by it. + assert "/dev/davinci3" in seen_paths + # The device without one is skipped: Device.index is the logic id, so + # standing it in for the physical id would address another NPU's node. + assert "/dev/davinci1" not in seen_paths diff --git a/tests/gpustack_runtime/detector/test_cambricon.py b/tests/gpustack_runtime/detector/test_cambricon.py index 1f8f9d0..2b0aed5 100644 --- a/tests/gpustack_runtime/detector/test_cambricon.py +++ b/tests/gpustack_runtime/detector/test_cambricon.py @@ -1,5 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import cambricon, pycndev +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.cambricon import CambriconDetector @@ -21,3 +31,538 @@ def test_get_topology(): det = CambriconDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pycndev, so the split is provable on a host with no Cambricon driver. # +# --------------------------------------------------------------------------- # + +_HANDLE_OFFSET = 100 +""" +Distance between a card's index and the handle the fake driver hands out. +cndevDevice_t is an int32 handle, i.e. indistinguishable from an index at the +Python level, so the fake offsets it: a detector passing an index where a handle +belongs indexes out of range instead of quietly working. +""" + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MLU590-M9" + memory_total: int = 49152 + memory_used: int = 1024 + driver_version: tuple[int, int, int] = (5, 10, 22) + bus_id: str = "0000:1f:00.0" + numa_node: int = 0 + # The header's `health` carries the same 0/1 convention as the device-state + # enumerators, and the operator's verdict is a bare `Health == 0`. + health: int = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + core_utilization: int = 42 + temperature: int = 55 + power_usage: int = 150 + failing: tuple[str, ...] = () + """ + Entry points that raise for this card, so a faulty card is reproducible. + """ + + +@dataclass +class _MemoryInfo: + """ + A c_cndevMemoryInfoV2_t stand-in. The header calls the unit MB; the operator, + cnmon and this repo's Ascend detector all treat it as MiB. + """ + + physicalMemoryTotal: int # noqa: N815 + physicalMemoryUsed: int # noqa: N815 + + +@dataclass +class _VersionInfo: + """ + A c_cndevVersionInfo_t stand-in, of which only the driver triplet is read. + """ + + driverMajorVersion: int # noqa: N815 + driverMinorVersion: int # noqa: N815 + driverBuildVersion: int # noqa: N815 + + +@dataclass +class _NUMANodeId: + """ + A c_cndevNUMANodeId_t stand-in. + """ + + nodeId: int # noqa: N815 + + +@dataclass +class _CardHealthState: + """ + A c_cndevCardHealthStateV2_t stand-in, of which the detector reads the + card-wide health bit alone, as the operator does. + """ + + health: int + deviceState: int # noqa: N815 + + +@dataclass +class _UtilizationInfo: + """ + A c_cndevUtilizationInfo_t stand-in. + """ + + averageCoreUtilization: int # noqa: N815 + + +@dataclass +class _TemperatureInfo: + """ + A c_cndevTemperatureInfo_t stand-in, in degrees Celsius. + """ + + chip: int + + +@dataclass +class _PowerInfo: + """ + A c_cndevDevicePowerInfo_t stand-in, in Watts. + """ + + usage: int + + +@dataclass +class _FakeCNDev: + """ + A stand-in for the pycndev binding, recording every call the detector makes. + + The error type, the error codes and the health enumerators are the real + module's, so a fake drifting from the binding's contract fails here rather + than on hardware. Entry points are dispatched by name, as the sibling vendors' + fakes do, which keeps the driver's camelCase out of the handlers' own names. + + cndevGetECCInfo is deliberately absent: the operator's Cambricon health + verdict reads the card health state and nothing else, so an ECC read + reintroduced here fails loudly instead of silently costing a driver call. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + CNDevError = pycndev.CNDevError + CNDEV_HEALTH_STATE_DEVICE_GOOD = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + + def __getattr__(self, name: str): + handler = { + "cndevInit": self._init, + "cndevGetDeviceCount": self._get_device_count, + "cndevGetDeviceHandleByIndex": self._get_device_handle_by_index, + "cndevGetUUID": self._get_uuid, + "cndevGetCardNameByDevId": self._get_card_name_by_dev_id, + "cndevGetMemoryUsageV2": self._get_memory_usage_v2, + "cndevGetVersionInfo": self._get_version_info, + "cndevGetPCIeBusId": self._get_pcie_bus_id, + "cndevGetNUMANodeIdByDevId": self._get_numa_node_id_by_dev_id, + "cndevGetCardHealthStateV2": self._get_card_health_state_v2, + "cndevGetDeviceUtilizationInfo": self._get_device_utilization_info, + "cndevGetTemperatureInfo": self._get_temperature_info, + "cndevGetDevicePowerInfo": self._get_device_power_info, + }.get(name) + if handler is None: + msg = f"module pycndev has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _card(self, handle: int, name: str) -> _Card: + card = self.cards[handle - _HANDLE_OFFSET] + if name in card.failing: + raise pycndev.CNDevError(pycndev.CNDEV_ERROR_UNKNOWN) + return card + + def _init(self) -> None: + pass + + def _get_device_count(self) -> int: + return len(self.cards) + + def _get_device_handle_by_index(self, index: int) -> int: + return index + _HANDLE_OFFSET + + def _get_uuid(self, handle: int) -> str: + # The binding prefixes the driver's bare UUID, as every Cambricon tool does. + return "MLU-" + self._card(handle, "cndevGetUUID").uuid + + def _get_card_name_by_dev_id(self, handle: int) -> str: + return self._card(handle, "cndevGetCardNameByDevId").name + + def _get_memory_usage_v2(self, handle: int) -> _MemoryInfo: + card = self._card(handle, "cndevGetMemoryUsageV2") + return _MemoryInfo( + physicalMemoryTotal=card.memory_total, + physicalMemoryUsed=card.memory_used, + ) + + def _get_version_info(self, handle: int) -> _VersionInfo: + card = self._card(handle, "cndevGetVersionInfo") + major, minor, build = card.driver_version + return _VersionInfo( + driverMajorVersion=major, + driverMinorVersion=minor, + driverBuildVersion=build, + ) + + def _get_pcie_bus_id(self, handle: int) -> str: + return self._card(handle, "cndevGetPCIeBusId").bus_id + + def _get_numa_node_id_by_dev_id(self, handle: int) -> _NUMANodeId: + card = self._card(handle, "cndevGetNUMANodeIdByDevId") + return _NUMANodeId(nodeId=card.numa_node) + + def _get_card_health_state_v2(self, handle: int) -> _CardHealthState: + card = self._card(handle, "cndevGetCardHealthStateV2") + return _CardHealthState(health=card.health, deviceState=card.health) + + def _get_device_utilization_info(self, handle: int) -> _UtilizationInfo: + card = self._card(handle, "cndevGetDeviceUtilizationInfo") + return _UtilizationInfo(averageCoreUtilization=card.core_utilization) + + def _get_temperature_info(self, handle: int) -> _TemperatureInfo: + card = self._card(handle, "cndevGetTemperatureInfo") + return _TemperatureInfo(chip=card.temperature) + + def _get_device_power_info(self, handle: int) -> _PowerInfo: + card = self._card(handle, "cndevGetDevicePowerInfo") + return _PowerInfo(usage=card.power_usage) + + +_USAGE_CALLS = ( + "cndevGetDeviceUtilizationInfo", + "cndevGetTemperatureInfo", + "cndevGetDevicePowerInfo", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: cndevGetMemoryUsageV2 +reports the memory *total* the inventory needs alongside the used amount, so +both queries read it, as the operator's DetectAccelerator and MonitorAccelerator +both do. +""" + +_MEMORY_UTILIZATION = 2.08 +""" +A default card's memory utilization: 1024 MiB used of 49152 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch, tmp_path): + """ + Build a Cambricon detector talking to a fake driver reporting the given cards. + """ + + def _install( + *cards: _Card, + neuware_version: str | None = "Neuware Version: 1.2.3\n", + ) -> tuple[CambriconDetector, _FakeCNDev]: + fake = _FakeCNDev(cards=list(cards)) + monkeypatch.setattr(cambricon, "pycndev", fake) + + # The Neuware version is a file read, not a driver call, so it is faked + # by pointing the module's path constant at a temporary file. + version_path = tmp_path / "version.txt" + if neuware_version is not None: + version_path.write_text(neuware_version) + monkeypatch.setattr(cambricon, "_NEUWARE_VERSION_PATH", version_path) + + det = CambriconDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: inventory only, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="0123456789ab")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.CAMBRICON + assert dev.index == 0 + assert dev.name == "MLU590-M9" + assert dev.uuid == "MLU-0123456789ab" + # The operator formats major.minor only; the build number is free precision + # from the same query, and it is the digit a driver bug is diagnosed by. + assert dev.driver_version == "5.10.22" + assert dev.runtime_version == "1.2" + assert dev.runtime_version_original == "1.2.3" + # The driver reports MB per the header and MiB in fact, so no conversion is + # applied: converting would under-report ~4.8% against the operator. + assert dev.memory == 49152 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:1f:00.0" + assert dev.appendix["numa"] == "0" + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + # No power call is made, so the power limit stays unset. + assert dev.power is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="0123456789ab")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + assert "cndevGetMemoryUsageV2" in fake.calls + assert "cndevGetVersionInfo" in fake.calls + + +def test_detect_info_enumerates_every_card(detector): + det, _ = detector(_Card(uuid="card-0"), _Card(uuid="card-1")) + + devices = det.detect_info() + + assert [dev.uuid for dev in devices] == ["MLU-card-0", "MLU-card-1"] + assert [dev.index for dev in devices] == [0, 1] + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector(_Card(uuid="card-0")) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +def test_detect_info_without_the_neuware_version_file(detector): + det, _ = detector(_Card(uuid="card-0"), neuware_version=None) + + dev = det.detect_info()[0] + + assert dev.runtime_version is None + assert dev.runtime_version_original is None + + +def test_detect_info_ignores_an_unversioned_neuware_file(detector): + det, _ = detector(_Card(uuid="card-0"), neuware_version="unknown\n") + + dev = det.detect_info()[0] + + assert dev.runtime_version_original is None + + +# --------------------------------------------------------------------------- # +# A faulty card is skipped, not fatal. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "failing", + ["cndevGetUUID", "cndevGetMemoryUsageV2", "cndevGetCardNameByDevId"], +) +def test_detect_info_skips_a_faulty_card(detector, failing): + det, _ = detector( + _Card(uuid="broken", failing=(failing,)), + _Card(uuid="healthy"), + ) + + devices = det.detect_info() + + # The operator continues past a card whose required reads fail. Without that, + # one faulty card makes the whole vendor report zero devices. + assert [dev.uuid for dev in devices] == ["MLU-healthy"] + # Skipping does not renumber: the index is the one the driver enumerated the + # card at, as Device.index promises. + assert devices[0].index == 1 + + +def test_detect_info_fails_when_every_card_is_skipped(detector): + # Skipping one card of several is the graceful degradation this vendor is + # allowed. Every card failing the same read is systemic -- a driver not + # exporting a call the loop needs -- and an empty inventory would be + # indistinguishable from a host that has no MLU at all. + det, _ = detector( + _Card(uuid="broken-0", failing=("cndevGetPCIeBusId",)), + _Card(uuid="broken-1", failing=("cndevGetPCIeBusId",)), + ) + + with pytest.raises(pycndev.CNDevError): + det.detect_info() + + +def test_detect_usage_fails_when_every_card_is_skipped(detector): + det, _ = detector( + _Card(uuid="broken-0", failing=("cndevGetMemoryUsageV2",)), + _Card(uuid="broken-1", failing=("cndevGetMemoryUsageV2",)), + ) + + with pytest.raises(pycndev.CNDevError): + det.detect_usage() + + +def test_detect_info_tolerates_an_unreadable_optional_field(detector): + det, _ = detector( + _Card( + uuid="card-0", + failing=("cndevGetVersionInfo", "cndevGetNUMANodeIdByDevId"), + ), + ) + + dev = det.detect_info()[0] + + # The optional reads are suppressed, so the card is still reported. + assert dev.uuid == "MLU-card-0" + assert dev.driver_version is None + assert "numa" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="card-0"), + _Card( + uuid="card-1", + memory_used=2048, + core_utilization=7, + temperature=61, + power_usage=200, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["MLU-card-0"].cores_utilization == 42 + assert by_uuid["MLU-card-0"].memory_used == 1024 + assert by_uuid["MLU-card-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["MLU-card-0"].temperature == 55 + assert by_uuid["MLU-card-0"].power_used == 150 + assert by_uuid["MLU-card-1"].cores_utilization == 7 + assert by_uuid["MLU-card-1"].memory_used == 2048 + assert by_uuid["MLU-card-1"].temperature == 61 + assert by_uuid["MLU-card-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["MLU-card-0"].index == 0 + assert by_uuid["MLU-card-0"].memory == 49152 + assert by_uuid["MLU-card-0"].name == "MLU590-M9" + assert by_uuid["MLU-card-0"].driver_version == "5.10.22" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="card-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["MLU-card-0"] + assert devices[0].name == "MLU590-M9" + assert devices[0].memory == 49152 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="card-0")) + + dev = det.detect()[0] + + assert dev.name == "MLU590-M9" + assert dev.memory == 49152 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +def test_detect_usage_skips_a_faulty_card(detector): + det, _ = detector( + _Card(uuid="broken", failing=("cndevGetMemoryUsageV2",)), + _Card(uuid="healthy"), + ) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["MLU-healthy"] + assert devices[0].memory_used == 1024 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="card-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reads_no_health_state_by_default(detector): + det, fake = detector(_Card(uuid="card-0")) + + det.detect() + + # GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults to true, so a default run + # pays for no health call at all -- a deliberate divergence from the + # operator, which reads the card health unconditionally. + assert "cndevGetCardHealthStateV2" not in fake.calls + + +def test_detect_reports_a_card_in_problem(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector( + _Card( + uuid="card-0", + health=pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM, + ), + ) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_tolerates_an_unreadable_health_state(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector( + _Card(uuid="card-0", failing=("cndevGetCardHealthStateV2",)), + ) + + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY diff --git a/tests/gpustack_runtime/detector/test_detector_cli.py b/tests/gpustack_runtime/detector/test_detector_cli.py new file mode 100644 index 0000000..c910ac4 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_detector_cli.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +from gpustack_runtime.cmds import detector as cmds_detector +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, +) + + +# --------------------------------------------------------------------------- # +# Helpers. # +# --------------------------------------------------------------------------- # +def _parse(*argv: str) -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + cmds_detector.DetectDevicesSubCommand.register(subparsers) + return parser.parse_args(["detect", *argv]) + + +def _device() -> Device: + return Device( + manufacturer=ManufacturerEnum.NVIDIA, + index=0, + name="NVIDIA A100-SXM4-40GB", + uuid="GPU-0", + driver_version="580.65.06", + runtime_version="13.0", + compute_capability="8.0", + cores=6912, + memory=40960, + memory_status=DeviceMemoryStatusEnum.HEALTHY, + appendix={}, + ) + + +# --------------------------------------------------------------------------- # +# `detect --no-usage` threading. # +# --------------------------------------------------------------------------- # +def test_detect_asks_for_usage_by_default(monkeypatch): + calls: list[dict] = [] + + def fake_detect_devices(**kwargs): + calls.append(kwargs) + return [] + + monkeypatch.setattr(cmds_detector, "detect_devices", fake_detect_devices) + + cmds_detector.DetectDevicesSubCommand(_parse("--format", "json")).run() + + assert calls == [{"fast": False, "usage": True}] + + +def test_detect_no_usage_skips_the_usage_query(monkeypatch): + calls: list[dict] = [] + + def fake_detect_devices(**kwargs): + calls.append(kwargs) + return [] + + monkeypatch.setattr(cmds_detector, "detect_devices", fake_detect_devices) + + cmds_detector.DetectDevicesSubCommand( + _parse("--no-usage", "--format", "json"), + ).run() + + # The flag is the only thing standing between the CLI and a metric call: + # every detector's usage query is skipped when `usage` is false. + assert calls == [{"fast": False, "usage": False}] + + +# --------------------------------------------------------------------------- # +# Table rendering: `N/A` for what was never measured. # +# --------------------------------------------------------------------------- # +def test_table_renders_measured_usage_as_numbers(): + dev = _device() + dev.cores_utilization = 42 + dev.memory_used = 1024 + dev.temperature = 61 + + table = cmds_detector.format_devices_table([dev]) + + assert "1024MiB / 40960MiB" in table + assert "42%" in table + assert "61C" in table + + +def test_table_renders_unmeasured_usage_as_not_available(): + table = cmds_detector.format_devices_table([_device()], usage=False) + + # A zero here is indistinguishable from a real idle reading, which is + # exactly the ambiguity `--no-usage` exists to remove. + assert "N/A / 40960MiB" in table + assert "0%" not in table + # The used-memory slot, i.e. everything left of the "/", carries no number. + assert not re.search(r"\d+MiB / ", table) + assert not re.search(r"\d+C", table) + # Information fields keep their values, and health is reported by both + # queries, so the status column stays meaningful. + assert "NVIDIA A100-SXM4-40GB" in table + assert "8.0" in table + assert "OK" in table + + +# --------------------------------------------------------------------------- # +# JSON rendering: an unmeasured field is absent, not zero. # +# --------------------------------------------------------------------------- # +def test_json_renders_measured_usage_as_numbers(): + dev = _device() + dev.cores_utilization = 42 + dev.memory_used = 1024 + dev.temperature = 61 + + payload = json.loads(cmds_detector.format_devices_json([dev])) + + assert payload[0]["cores_utilization"] == 42 + assert payload[0]["memory_used"] == 1024 + assert payload[0]["temperature"] == 61 + + +def test_json_omits_unmeasured_usage(): + dev = _device() + dev.appendix = {"mig_devices": [{"uuid": "MIG-0", "memory_used": 0}]} + + payload = json.loads(cmds_detector.format_devices_json([dev], usage=False)) + + # A serialized 0 is indistinguishable from a real idle reading, which is + # what `--no-usage` exists to remove -- and a machine-readable consumer is + # the more likely of the two to act on it. + for key in ("cores_utilization", "memory_used", "memory_utilization"): + assert key not in payload[0] + for key in ("temperature", "power_used"): + assert key not in payload[0] + # MIG instances carry the same fields, so they are dropped there too. + assert "memory_used" not in payload[0]["appendix"]["mig_devices"][0] + # Information fields keep their values, and health is reported by both + # queries, so the status stays meaningful. + assert payload[0]["memory"] == 40960 + assert payload[0]["memory_status"] == DeviceMemoryStatusEnum.HEALTHY + + +# --------------------------------------------------------------------------- # +# Tree-wide guard: no detector can emit a `vgpu` appendix key. # +# --------------------------------------------------------------------------- # +def test_no_detector_emits_a_vgpu_appendix_key(): + package = Path(cmds_detector.__file__).parent.parent + offenders = [ + str(path.relative_to(package)) + for path in sorted(package.rglob("*.py")) + if re.search(r"""['"]vgpu['"]""", path.read_text(encoding="utf-8")) + ] + + # The vGPU/virtual-card classification is gone for every vendor, so no + # shipped module may name the appendix key again. The vendor bindings' + # own `*_VGPU_*` constants are upstream SDK symbols, not appendix keys, + # and do not match. + assert offenders == [] diff --git a/tests/gpustack_runtime/detector/test_detector_types.py b/tests/gpustack_runtime/detector/test_detector_types.py new file mode 100644 index 0000000..febb5e0 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_detector_types.py @@ -0,0 +1,422 @@ +from __future__ import annotations + +import pytest + +from gpustack_runtime import envs +from gpustack_runtime.detector import ( + _DETECTORS, + Device, + DeviceMemoryStatusEnum, + Devices, + ManufacturerEnum, +) +from gpustack_runtime.detector.__types__ import ( + Detector, + merge_devices_usage, +) +from gpustack_runtime.detector.__utils__ import ( + _load_pci_device_names, + get_pci_device_name, +) + +# --------------------------------------------------------------------------- # +# Detector ABC: detect_info / detect_usage / detect composition. # +# --------------------------------------------------------------------------- # + + +class _RecordingDetector(Detector): + """ + A detector recording which query it was asked for, as the vendors behave + during the expand step: detect_info returns everything, detect_usage + merges a usage payload in. + """ + + def __init__( + self, + devices: Devices | None = None, + usages: Devices | None = None, + ): + super().__init__(ManufacturerEnum.UNKNOWN) + self.devices = devices + self.usages = usages + self.calls: list[str] = [] + + @staticmethod + def is_supported() -> bool: + return True + + def detect_info(self) -> Devices | None: + self.calls.append("info") + return self.devices + + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + self.calls.append("usage") + if devices is None: + devices = self.detect_info() + return merge_devices_usage(devices, self.usages) + + +def _device(uuid: str, **kwargs) -> Device: + device = Device( + manufacturer=ManufacturerEnum.NVIDIA, + index=0, + name="NVIDIA A100-SXM4-40GB", + uuid=uuid, + cores=6912, + memory=40960, + appendix={}, + ) + for key, value in kwargs.items(): + setattr(device, key, value) + return device + + +def _usage(uuid: str, cores_utilization: int = 42) -> Device: + return Device( + uuid=uuid, + cores_utilization=cores_utilization, + memory_used=1024, + memory_utilization=2.5, + memory_status=DeviceMemoryStatusEnum.UNHEALTHY, + temperature=61, + power_used=250, + ) + + +def test_detect_composes_information_then_usage(): + det = _RecordingDetector(devices=[_device("GPU-0")], usages=[_usage("GPU-0")]) + + devices = det.detect() + + assert det.calls == ["info", "usage"] + assert devices[0].cores_utilization == 42 + assert devices[0].power_used == 250 + + +def test_detect_skips_the_usage_query_when_not_asked(): + det = _RecordingDetector(devices=[_device("GPU-0")], usages=[_usage("GPU-0")]) + + devices = det.detect(usage=False) + + assert det.calls == ["info"] + # The information query owns no usage field, so they keep their defaults. + assert devices[0].cores_utilization == 0 + assert devices[0].power_used is None + + +def test_detect_reports_the_inventory_when_the_usage_query_fails(): + # A card exists whether or not its metrics can be read. detect_devices logs + # a raising detector and moves on, so letting this propagate would report no + # hardware at all on a host whose inventory query just succeeded. + class _FailingUsageDetector(_RecordingDetector): + def detect_usage(self, devices: Devices | None = None) -> Devices | None: + self.calls.append("usage") + msg = "the driver lost a device mid-pass" + raise RuntimeError(msg) + + det = _FailingUsageDetector(devices=[_device("GPU-0")]) + + devices = det.detect() + + assert det.calls == ["info", "usage"] + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].cores_utilization == 0 + + +def test_detect_skips_the_usage_query_without_devices(): + for empty in (None, []): + det = _RecordingDetector(devices=empty) + + assert det.detect() == empty + assert det.calls == ["info"] + + +@pytest.mark.parametrize( + "name, body", + [ + # Neither query implemented. + ("_NoQueryDetector", {}), + # Only the information query: a vendor cannot keep a permissive usage + # default that makes detect() look like it measured something. + ("_InfoOnlyDetector", {"detect_info": lambda _self: None}), + # Only the usage query. + ("_UsageOnlyDetector", {"detect_usage": lambda _self, devices=None: devices}), + ], +) +def test_a_detector_skipping_the_split_cannot_be_instantiated(name, body): + detector_type = type( + name, + (Detector,), + {"is_supported": staticmethod(lambda: True), **body}, + ) + + # Both queries are abstract, so a future vendor fails at construction + # rather than silently returning an unmeasured device at run time. + with pytest.raises(TypeError): + detector_type(ManufacturerEnum.UNKNOWN) + + +def test_a_detector_implementing_both_queries_can_be_instantiated(): + assert _RecordingDetector().detect() is None + + +# --------------------------------------------------------------------------- # +# merge_devices_usage. # +# --------------------------------------------------------------------------- # + + +def test_merge_devices_usage_matches_by_uuid(): + devices = [_device("GPU-0"), _device("GPU-1")] + + # Reversed *and* told apart by their payload: the operator's monitor pass + # returns its own list and every consumer joins it by device identity, + # never by index. Identical payloads would leave a positional join, a + # reversed mapping and a broadcast-to-every-card indistinguishable, so the + # values have to differ for this to pin anything. + merge_devices_usage( + devices, + [_usage("GPU-1", cores_utilization=7), _usage("GPU-0", cores_utilization=42)], + ) + + assert [dev.cores_utilization for dev in devices] == [42, 7] + assert [dev.memory_used for dev in devices] == [1024, 1024] + assert [dev.temperature for dev in devices] == [61, 61] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.UNHEALTHY, + DeviceMemoryStatusEnum.UNHEALTHY, + ] + + +def test_merge_devices_usage_keeps_a_health_verdict_the_usage_never_read(): + # No vendor's health helper returns UNKNOWN -- they answer HEALTHY or + # UNHEALTHY -- so an entry carrying UNKNOWN never read health, and writing it + # would erase the information query's verdict. The CLI renders UNKNOWN as + # ERR too, so the erasure is not even visible as one. + devices = [_device("GPU-0", memory_status=DeviceMemoryStatusEnum.UNHEALTHY)] + usage = _usage("GPU-0") + usage.memory_status = DeviceMemoryStatusEnum.UNKNOWN + + merge_devices_usage(devices, [usage]) + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + # Every other usage field still merges. + assert devices[0].cores_utilization == 42 + assert devices[0].temperature == 61 + + +def test_merge_devices_usage_downgrades_a_health_verdict_the_usage_did_read(): + devices = [_device("GPU-0", memory_status=DeviceMemoryStatusEnum.UNHEALTHY)] + usage = _usage("GPU-0") + usage.memory_status = DeviceMemoryStatusEnum.HEALTHY + + merge_devices_usage(devices, [usage]) + + # A card that recovered must be allowed to say so. + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_merge_devices_usage_drops_an_ambiguous_uuid(): + # A driver answering the same id for every card -- rocm-smi's unique id + # reporting 0, say -- leaves two cards indistinguishable. Writing either + # card's metrics onto both is worse than reporting neither, so the merge + # leaves them with what the information query read. + devices = [_device("GPU-0"), _device("GPU-0")] + + merge_devices_usage( + devices, + [_usage("GPU-0", cores_utilization=7), _usage("GPU-0", cores_utilization=42)], + ) + + assert [dev.cores_utilization for dev in devices] == [0, 0] + assert [dev.memory_used for dev in devices] == [0, 0] + + +def test_merge_devices_usage_keeps_the_information_fields(): + devices = [_device("GPU-0")] + usage = _usage("GPU-0") + usage.name = "wrong" + usage.memory = 1 + usage.cores = 1 + usage.index = 7 + + merge_devices_usage(devices, [usage]) + + assert devices[0].name == "NVIDIA A100-SXM4-40GB" + assert devices[0].memory == 40960 + assert devices[0].cores == 6912 + assert devices[0].index == 0 + + +def test_merge_devices_usage_merges_the_mig_devices(): + mig_device = { + "index": 2, + "name": "1g.5gb", + "uuid": "MIG-0-0", + "memory": 4864, + "cores_utilization": 0, + "memory_used": 0, + "memory_status": DeviceMemoryStatusEnum.HEALTHY, + "appendix": {"sliced": True, "mig": True}, + } + devices = [_device("GPU-0", appendix={"mig": True, "mig_devices": [mig_device]})] + + merge_devices_usage(devices, [_usage("MIG-0-0"), _usage("GPU-0")]) + + assert mig_device["cores_utilization"] == 42 + assert mig_device["memory_used"] == 1024 + assert mig_device["temperature"] == 61 + assert mig_device["memory_status"] == DeviceMemoryStatusEnum.UNHEALTHY + # The MIG entry stays an appendix entry, and its own fields stay put. + assert mig_device["name"] == "1g.5gb" + assert mig_device["memory"] == 4864 + assert devices[0].cores_utilization == 42 + + +def test_merge_devices_usage_ignores_an_unknown_uuid(): + devices = [_device("GPU-0")] + + merge_devices_usage(devices, [_usage("GPU-9")]) + + assert devices[0].cores_utilization == 0 + assert devices[0].power_used is None + + +def test_merge_devices_usage_tolerates_nothing_to_merge(): + devices = [_device("GPU-0", appendix=None)] + + assert merge_devices_usage(devices, None) is devices + assert merge_devices_usage(devices, []) is devices + assert merge_devices_usage(None, [_usage("GPU-0")]) is None + + +# --------------------------------------------------------------------------- # +# The vendor detectors, as the expand step leaves them. # +# --------------------------------------------------------------------------- # + + +def test_every_detector_implements_the_split(): + for det in _DETECTORS: + # No vendor overrides detect: its payload is detect_info's, plus a + # usage merge that is a no-op until the vendor migrates. That is what + # keeps detect()'s output identical to the pre-split one. + assert type(det).detect is Detector.detect, det.name + assert type(det).detect_info is not Detector.detect_info, det.name + + +def test_every_detector_accepts_no_usage(): + for det in _DETECTORS: + devices = det.detect(usage=False) + assert devices is None or isinstance(devices, list), det.name + + +def test_no_physical_index_switch_survives(): + # Device.index is the enumeration index now, with no switch to make it the + # driver-physical number. Asserted on the whole environment surface rather + # than on the retired name, so a renamed reincarnation fails as well. + assert [name for name in dir(envs) if "PHYSICAL_INDEX" in name] == [] + + +# --------------------------------------------------------------------------- # +# get_pci_device_name: the pci.ids lookup mirroring the operator's # +# GetPCIDeviceNames / GetName. # +# --------------------------------------------------------------------------- # + +# An extract of the real pci.ids, keeping its exact shape: a class section, a +# comment, vendor lines at column 0, device lines behind one tab, subsystem +# lines behind two. +_PCI_IDS = """\ +#\tList of PCI ID's +# +1002 Advanced Micro Devices, Inc. [AMD/ATI] +\t744c Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M] +\t\t1002 0e0d Radeon RX 7900 XTX +\t\t1eae 7900 RX 7900 XTX Phantom Gaming +\t74a1 Aqua Vanjaram [Instinct MI300X] +1d94 Chengdu Haiguang IC Design Co., Ltd. +\t6210 Kunpeng [K100 AI] +\t\t1d94 6210 K100_AI +10de NVIDIA Corporation +\t2330 GH100 [H100 SXM5 80GB] +C 03 Display controller +\t00 VGA compatible controller +""" + + +@pytest.fixture +def pci_ids(tmp_path, monkeypatch): + """ + Point the pci.ids lookup at a fixture database. + """ + + def _write(content: str | None = _PCI_IDS) -> None: + paths: tuple[str, ...] = () + if content is not None: + path = tmp_path / "pci.ids" + path.write_text(content, encoding="utf-8") + paths = (str(path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + paths, + ) + _load_pci_device_names.cache_clear() + + yield _write + + _load_pci_device_names.cache_clear() + + +def test_get_pci_device_name_resolves_a_vendor_device_pair(pci_ids): + pci_ids() + + assert get_pci_device_name("1002", "74a1") == "Aqua Vanjaram [Instinct MI300X]" + assert get_pci_device_name("1d94", "6210") == "Kunpeng [K100 AI]" + assert get_pci_device_name("10de", "2330") == "GH100 [H100 SXM5 80GB]" + + +def test_get_pci_device_name_normalizes_the_ids(pci_ids): + pci_ids() + + # sysfs hands out "0x1002", the SMI libraries hand out an int, and pci.ids + # is lowercase hexadecimal. + for vendor, device in ( + ("0x1002", "0x74A1"), + ("1002", "74A1"), + (0x1002, 0x74A1), + ): + assert get_pci_device_name(vendor, device) == "Aqua Vanjaram [Instinct MI300X]" + + +def test_get_pci_device_name_prefers_the_subsystem_name(pci_ids): + pci_ids() + + assert ( + get_pci_device_name("1002", "744c", "1eae", "7900") + == "RX 7900 XTX Phantom Gaming" + ) + # An unknown subsystem falls back to the device name, as the operator's + # GetName does. + assert ( + get_pci_device_name("1002", "744c", "1eae", "0000") + == "Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M]" + ) + assert ( + get_pci_device_name("1002", "744c") + == "Navi 31 [Radeon RX 7900 XT/7900 XTX/7900M]" + ) + + +def test_get_pci_device_name_returns_nothing_when_unknown(pci_ids): + pci_ids() + + assert get_pci_device_name("1002", "ffff") == "" + assert get_pci_device_name("ffff", "744c") == "" + assert get_pci_device_name("", "") == "" + # The device class table trailing the vendors closes the vendor section: + # its entries must not be attributed to the last vendor parsed. + assert get_pci_device_name("10de", "00") == "" + + +def test_get_pci_device_name_tolerates_a_missing_database(pci_ids): + pci_ids(content=None) + + assert get_pci_device_name("1002", "74a1") == "" diff --git a/tests/gpustack_runtime/detector/test_hygon.py b/tests/gpustack_runtime/detector/test_hygon.py index d304ecc..db37bc0 100644 --- a/tests/gpustack_runtime/detector/test_hygon.py +++ b/tests/gpustack_runtime/detector/test_hygon.py @@ -1,5 +1,23 @@ +from __future__ import annotations + +import contextlib +from types import SimpleNamespace + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import hygon as cdi_hygon +from gpustack_runtime.deployer.cdi.__types__ import ConfigDeviceNode +from gpustack_runtime.detector import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, + amd, + hygon, + pyamdgpu, + pyhsa, +) +from gpustack_runtime.detector.__utils__ import _load_pci_device_names from gpustack_runtime.detector.hygon import HygonDetector @@ -21,3 +39,483 @@ def test_get_topology(): det = HygonDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# Fake bindings: no Hygon driver exists in this suite, so every behaviour # +# below is proved against stand-ins for pyrocmsmi / pyhsa, recording the calls # +# they serve -- "detect_info asks for no usage" is a statement about the calls # +# made, not about the values returned. # +# --------------------------------------------------------------------------- # + + +class _FakeRocmSmiError(Exception): + """ + The fake ROCm SMI's own error type. + """ + + +class _EccCount: + """ + An rsmi_error_count_t stand-in. + """ + + def __init__(self, uncorrectable_err: int = 0): + self.uncorrectable_err = uncorrectable_err + + +class _FakeRocmSmi: + """ + A pyrocmsmi stand-in answering per device index out of the fixture cards. + """ + + ROCMSMIError = _FakeRocmSmiError + + def __init__(self, calls: list[str], cards: list[dict]): + self.calls = calls + self.cards = cards + + def rsmi_init(self, *_args): + self.calls.append("rsmi_init") + + def rsmi_get_rocm_version(self) -> str: + return "25.04.1" + + def rsmi_num_monitor_devices(self) -> int: + return len(self.cards) + + def rsmi_dev_unique_id_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["unique_id"] + + def rsmi_dev_pci_id_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["bdf"] + + def rsmi_dev_name_get(self, dev_idx: int) -> str: + self.calls.append("rsmi_dev_name_get") + return self.cards[dev_idx]["name"] + + def rsmi_dev_target_graphics_version_get(self, dev_idx: int) -> str: + return self.cards[dev_idx]["target_graphics_version"] + + def rsmi_dev_memory_total_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_memory_total_get") + return self.cards[dev_idx]["memory_total"] + + def rsmi_dev_memory_usage_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_memory_usage_get") + return self.cards[dev_idx]["memory_used"] + + def rsmi_dev_ecc_count_get(self, dev_idx: int) -> _EccCount: + self.calls.append("rsmi_dev_ecc_count_get") + return _EccCount(self.cards[dev_idx]["uncorrectable_err"]) + + def rsmi_dev_busy_percent_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_busy_percent_get") + return self.cards[dev_idx]["busy_percent"] + + def rsmi_dev_temp_metric_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_temp_metric_get") + return self.cards[dev_idx]["temperature"] + + def rsmi_dev_power_cap_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_cap_get") + return self.cards[dev_idx]["power_cap"] + + def rsmi_dev_power_get(self, dev_idx: int) -> int: + self.calls.append("rsmi_dev_power_get") + return self.cards[dev_idx]["power_used"] + + def rsmi_topo_get_numa_node_number(self, dev_idx: int) -> int: + return self.cards[dev_idx]["numa"] + + +class _FakeAmdGpu: + """ + A libdrm stand-in exposing only what the name precedence reaches for. + + The real call answers with a pointer into libdrm's own table, and NULL for + a device id the table does not carry, which the binding reports as "". + """ + + def __init__(self, marketing_name: str): + self._marketing_name = marketing_name + + def __getattr__(self, name): + # Everything not faked here comes from the real binding, so the fake + # cannot drift from it. + return getattr(pyamdgpu, name) + + @contextlib.contextmanager + def amdgpu_device(self, card): + yield SimpleNamespace(card=card) + + def amdgpu_get_marketing_name(self, _device) -> str: + return self._marketing_name + + def amdgpu_query_gpu_info(self, _device): + return SimpleNamespace(cu_active_number=0, family_id=0) + + +class _FakeHSA: + """ + A pyhsa stand-in returning the fixture agents. + """ + + Agent = pyhsa.Agent + + def __init__(self, agents: list): + self.agents = agents + + def get_agents(self) -> list: + return self.agents + + +# An extract of the real pci.ids: the board's name lives on the subsystem line, +# which is what the operator's GetName prefers. +_PCI_IDS = """\ +1d94 Chengdu Haiguang IC Design Co., Ltd. +\t6210 Kunpeng +\t\t1d94 6210 K100_AI +""" + +_PCI_IDS_NAME = "K100_AI" +_HSA_NAME = "Hygon K100 AI" +_MARKETING_NAME = "Hygon DCU K100 AI" +_RSMI_NAME = "Kunpeng" + +_MEMORY_TOTAL_BYTES = 68719476736 # 65536 MiB +_MEMORY_TOTAL = 65536 + + +def _card( + bdf: str, + unique_id: str, + *, + memory_used: int = 1073741824, # 1024 MiB + busy_percent: int = 44, + temperature: int = 51, + power_used: int = 217, + uncorrectable_err: int = 0, +) -> dict: + """ + One fake Hygon card: the answers the fake ROCm SMI serves for its index, + and the PCI IDs the fixture sysfs tree exposes for its BDF. + """ + return { + "bdf": bdf, + "pci_ids": { + "vendor": "0x1d94", + "device": "0x6210", + "subsystem_vendor": "0x1d94", + "subsystem_device": "0x6210", + }, + "unique_id": unique_id, + "name": _RSMI_NAME, + "target_graphics_version": "gfx936", + "memory_total": _MEMORY_TOTAL_BYTES, + "memory_used": memory_used, + "uncorrectable_err": uncorrectable_err, + "busy_percent": busy_percent, + "temperature": temperature, + "power_cap": 350, + "power_used": power_used, + "numa": 0, + } + + +def _agent(bdf: str) -> pyhsa.Agent: + return pyhsa.Agent( + device_type=1, + device_id="0x6210", + bdf=bdf, + uuid="", + name=_HSA_NAME, + compute_capability="gfx936", + compute_units=104, + ) + + +@pytest.fixture +def hygon_bindings(monkeypatch, tmp_path): + """ + Drive the Hygon detector off the fake bindings, a fixture sysfs PCI tree + and a fixture pci.ids database, returning the shared call log. + """ + + def _setup( + cards: list[dict], + agents: list | None = None, + pci_ids: str | None = _PCI_IDS, + ) -> list[str]: + calls: list[str] = [] + + monkeypatch.setattr(HygonDetector, "is_supported", staticmethod(lambda: True)) + monkeypatch.setattr(hygon, "pyrocmsmi", _FakeRocmSmi(calls, cards)) + monkeypatch.setattr(hygon, "pyhsa", _FakeHSA(list(agents or []))) + + pci_devices_path = tmp_path / "pci_devices" + for card in cards: + card_path = pci_devices_path / card["bdf"] + card_path.mkdir(parents=True) + for name, value in card["pci_ids"].items(): + (card_path / name).write_text(f"{value}\n") + # The pci.ids lookup is the AMD module's -- Hygon shares it, as it + # already shares the architecture family mapping. + monkeypatch.setattr(amd, "_PCI_DEVICES_PATH", pci_devices_path) + + pci_ids_paths: tuple[str, ...] = () + if pci_ids is not None: + pci_ids_path = tmp_path / "pci.ids" + pci_ids_path.write_text(pci_ids, encoding="utf-8") + pci_ids_paths = (str(pci_ids_path),) + monkeypatch.setattr( + "gpustack_runtime.detector.__utils__._PCI_IDS_PATHS", + pci_ids_paths, + ) + _load_pci_device_names.cache_clear() + + return calls + + yield _setup + + _load_pci_device_names.cache_clear() + + +# --------------------------------------------------------------------------- # +# detect_info: the device name's precedence, the inventory, and the usage # +# calls it must not make. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_pci_ids_name(hygon_bindings): + # The operator resolves the board's name from pci.ids before asking the + # driver, which only knows the chip. + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_PCI_IDS_NAME] + + +def test_detect_info_falls_back_to_the_hsa_product_name(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + pci_ids=None, + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_HSA_NAME] + + +def test_detect_info_falls_back_to_the_rocm_smi_name(hygon_bindings): + hygon_bindings([_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], pci_ids=None) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_RSMI_NAME] + + +def test_detect_info_asks_libdrm_for_the_marketing_name(hygon_bindings, monkeypatch): + # The same libdrm step AMD gained: between the HSA name and the driver + # name, the operator asks for the board's marketing name. + hygon_bindings([_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], pci_ids=None) + monkeypatch.setattr(hygon, "_get_card_and_renderd_id", lambda _bdf: (1, 128)) + monkeypatch.setattr( + hygon, + "pyamdgpu", + _FakeAmdGpu(marketing_name=_MARKETING_NAME), + ) + + devices = HygonDetector().detect_info() + + assert [dev.name for dev in devices] == [_MARKETING_NAME] + + +def test_detect_info_reports_the_inventory(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + dev = devices[0] + assert dev.manufacturer is ManufacturerEnum.HYGON + assert dev.index == 0 + assert dev.uuid == "GPU-9f8e7d6c5b4a3921" + assert dev.runtime_version == "25.04" + assert dev.runtime_version_original == "25.04.1" + assert dev.compute_capability == "gfx936" + assert dev.cores == 104 + assert dev.memory == _MEMORY_TOTAL + assert dev.memory_status is DeviceMemoryStatusEnum.HEALTHY + # The power limit is inventory, not usage. + assert dev.power == 350 + assert dev.appendix["bdf"] == "0000:0b:00.0" + assert dev.appendix["numa"] == "0" + + +def test_detect_reports_no_vgpu(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + det = HygonDetector() + + # There is no virtual/PF/VF classification any more, in either query. + assert "vgpu" not in det.detect_info()[0].appendix + assert "vgpu" not in det.detect()[0].appendix + + +def test_detect_info_issues_no_usage_call(hygon_bindings): + calls = hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert "rsmi_dev_busy_percent_get" not in calls + assert "rsmi_dev_temp_metric_get" not in calls + assert "rsmi_dev_memory_usage_get" not in calls + assert "rsmi_dev_power_get" not in calls + # The power limit is inventory, so its own call stays. + assert "rsmi_dev_power_cap_get" in calls + + dev = devices[0] + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_reports_unhealthy_memory(hygon_bindings, monkeypatch): + # The ECC read is opt-in, as the health check is disabled by default. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921", uncorrectable_err=3)], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_info() + + assert devices[0].memory_status is DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# detect_usage. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(hygon_bindings): + hygon_bindings( + [ + _card("0000:0b:00.0", "0x9f8e7d6c5b4a3921"), + _card( + "0000:0c:00.0", + "0x9f8e7d6c5b4a3922", + memory_used=8589934592, # 8192 MiB + busy_percent=87, + temperature=63, + power_used=298, + ), + ], + agents=[_agent("0000:0b:00.0"), _agent("0000:0c:00.0")], + ) + det = HygonDetector() + devices = det.detect_info() + + # Reversed, to prove the merge joins by UUID rather than by position. + det.detect_usage(list(reversed(devices))) + + assert [dev.cores_utilization for dev in devices] == [44, 87] + assert [dev.temperature for dev in devices] == [51, 63] + assert [dev.memory_used for dev in devices] == [1024, 8192] + assert [dev.power_used for dev in devices] == [217, 298] + assert [dev.memory_utilization for dev in devices] == [1.56, 12.5] + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + DeviceMemoryStatusEnum.HEALTHY, + ] + # The information fields survive the merge. + assert [dev.memory for dev in devices] == [_MEMORY_TOTAL, _MEMORY_TOTAL] + assert [dev.power for dev in devices] == [350, 350] + + +def test_detect_usage_detects_the_information_first(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect_usage() + + assert devices[0].name == _PCI_IDS_NAME + assert devices[0].cores_utilization == 44 + assert devices[0].power_used == 217 + + +def test_detect_composes_the_information_and_the_usage(hygon_bindings): + hygon_bindings( + [_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921")], + agents=[_agent("0000:0b:00.0")], + ) + + devices = HygonDetector().detect() + + assert devices[0].memory == _MEMORY_TOTAL + assert devices[0].memory_used == 1024 + assert devices[0].power == 350 + assert devices[0].power_used == 217 + + +# --------------------------------------------------------------------------- # +# The CDI generator, which numbers its device nodes from the appendix. # +# --------------------------------------------------------------------------- # + + +def _fake_device_node(path: str, **_kwargs) -> ConfigDeviceNode: + """ + Stand in for the host's device nodes: neither /dev/kfd nor /dev/dri exists + in this suite, so the real lookup would drop every path. + """ + return ConfigDeviceNode(path=path, type_="c") + + +def test_cdi_spec_numbers_the_device_nodes_from_the_appendix(monkeypatch): + monkeypatch.setattr(cdi_hygon, "device_to_cdi_device_node", _fake_device_node) + + config = cdi_hygon.HygonGenerator().generate( + [ + Device( + manufacturer=ManufacturerEnum.HYGON, + index=0, + name=_PCI_IDS_NAME, + uuid="GPU-9f8e7d6c5b4a3921", + appendix={"card_id": 3, "renderd_id": 131}, + ), + ], + ) + + # The DRM numbering comes from the appendix, never from Device.index -- + # here the card is enumerated at 0 and its nodes are card3 / renderD131. + device_nodes = config["devices"][0]["containerEdits"]["deviceNodes"] + assert [node["path"] for node in device_nodes] == [ + "/dev/dri/card3", + "/dev/dri/renderD131", + ] + assert [dev["name"] for dev in config["devices"]] == [ + "0", + "GPU-9f8e7d6c5b4a3921", + "all", + ] + assert [node["path"] for node in config["containerEdits"]["deviceNodes"]] == [ + "/dev/kfd", + "/dev/mkfd", + ] diff --git a/tests/gpustack_runtime/detector/test_iluvatar.py b/tests/gpustack_runtime/detector/test_iluvatar.py index f529912..999141a 100644 --- a/tests/gpustack_runtime/detector/test_iluvatar.py +++ b/tests/gpustack_runtime/detector/test_iluvatar.py @@ -1,5 +1,12 @@ +from __future__ import annotations + import pytest +from gpustack_runtime import envs +from gpustack_runtime.deployer.cdi import iluvatar as cdi_iluvatar +from gpustack_runtime.deployer.cdi.iluvatar import IluvatarGenerator +from gpustack_runtime.detector import Device, ManufacturerEnum, iluvatar +from gpustack_runtime.detector.__utils__ import byte_to_mebibyte from gpustack_runtime.detector.iluvatar import IluvatarDetector @@ -21,3 +28,306 @@ def test_get_topology(): det = IluvatarDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pyixml carrying a call log, so a test can assert exactly which # +# driver calls detect_info/detect_usage make -- several acceptance criteria # +# are "issues no metric call", which the returned values alone cannot prove. # +# --------------------------------------------------------------------------- # + + +class _FakeNVMLError(Exception): + """ + Stand-in for pyixml.NVMLError. + """ + + +class _FakeHandle: + def __init__(self, index: int): + self.index = index + + +class _FakeMemoryInfo: + def __init__(self, total: int, used: int): + self.total = total + self.used = used + + +class _FakeUtilizationRates: + def __init__(self, gpu: int): + self.gpu = gpu + + +class _FakePciInfo: + def __init__(self, bus_id: str): + self.busIdLegacy = bus_id + + +class FakePyixml: + """ + A fake pyixml binding, standing in for the real ctypes module so the + Iluvatar detector can be exercised on a machine with no IXML driver. + """ + + NVMLError = _FakeNVMLError + IXML_HEALTH_OK = 0 + NVML_TEMPERATURE_GPU = 0 + NVML_AFFINITY_SCOPE_NODE = 0 + nvmlMemory_v2 = 0x02000028 # noqa: N815 + + def __init__(self, *, device_count: int = 2, v2_memory: bool = True): + self.calls: list[str] = [] + self.device_count = device_count + self.v2_memory = v2_memory + # Deliberately different from the v1 numbers, so a test can tell + # which accessor a returned value came from. + self.v2_memory_total = 32 * 1024**3 + self.v2_memory_used = 8 * 1024**3 + self.v1_memory_total = 16 * 1024**3 + self.v1_memory_used = 4 * 1024**3 + + # The method names below mirror pyixml's real (camelCase) API one-for-one, + # so a test can monkeypatch this in as a drop-in for the module. + def nvmlInit(self): # noqa: N802 + self.calls.append("nvmlInit") + + def nvmlSystemGetDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetDriverVersion") + return "4.2.0" + + def nvmlSystemGetCudaDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetCudaDriverVersion") + return 10020 + + def nvmlDeviceGetCount(self): # noqa: N802 + self.calls.append("nvmlDeviceGetCount") + return self.device_count + + def nvmlDeviceGetHandleByIndex(self, index): # noqa: N802 + self.calls.append("nvmlDeviceGetHandleByIndex") + return _FakeHandle(index) + + def nvmlDeviceGetMinorNumber(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetMinorNumber") + return 10 + dev.index + + def nvmlDeviceGetName(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetName") + return "Iluvatar BI-V150" + + def nvmlDeviceGetUUID(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetUUID") + return f"GPU-{dev.index}" + + def nvmlDeviceGetNumGpuCores(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetNumGpuCores") + return 4096 + + def nvmlDeviceGetMemoryInfo(self, dev, version=None): # noqa: N802 + if version: + self.calls.append("nvmlDeviceGetMemoryInfo_v2") + if not self.v2_memory: + msg = "v2 memory info not supported" + raise self.NVMLError(msg) + return _FakeMemoryInfo(self.v2_memory_total, self.v2_memory_used) + self.calls.append("nvmlDeviceGetMemoryInfo") + return _FakeMemoryInfo(self.v1_memory_total, self.v1_memory_used) + + def ixmlDeviceGetHealth(self, dev): # noqa: N802 + self.calls.append("ixmlDeviceGetHealth") + return self.IXML_HEALTH_OK + + def nvmlDeviceGetPowerManagementDefaultLimit(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerManagementDefaultLimit") + return 250_000 + + def nvmlDeviceGetPowerUsage(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerUsage") + return 90_000 + + def nvmlDeviceGetCudaComputeCapability(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetCudaComputeCapability") + return (7, 0) + + def nvmlDeviceGetPciInfo(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetPciInfo") + return _FakePciInfo(f"0000:0{dev.index}:00.0") + + def nvmlDeviceGetMemoryAffinity(self, dev, size, scope): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryAffinity") + return [0] + + def nvmlDeviceGetUtilizationRates(self, dev): # noqa: N802 + self.calls.append("nvmlDeviceGetUtilizationRates") + return _FakeUtilizationRates(37) + + def nvmlDeviceGetTemperature(self, dev, sensor): # noqa: N802 + self.calls.append("nvmlDeviceGetTemperature") + return 55 + + +@pytest.fixture(autouse=True) +def _reset_is_supported_cache(): + # is_supported()/detect_pci_devices() are lru_cache'd, so a value + # observed by one test would otherwise leak into the next. + IluvatarDetector.is_supported.cache_clear() + IluvatarDetector.detect_pci_devices.cache_clear() + yield + IluvatarDetector.is_supported.cache_clear() + IluvatarDetector.detect_pci_devices.cache_clear() + + +@pytest.fixture +def fake_pyixml(monkeypatch): + def _install(**kwargs) -> FakePyixml: + fake = FakePyixml(**kwargs) + monkeypatch.setattr(iluvatar, "pyixml", fake) + # No PCI sysfs tree exists on the dev machine, so bypass the PCI + # presence check that is_supported() otherwise gates on. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_PCI_CHECK", True) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: memory V2->V1 fallback, no vgpu, no usage calls. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_v2_memory_over_v1(fake_pyixml): + fake = fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert [dev.memory for dev in devices] == [ + byte_to_mebibyte(fake.v2_memory_total), + ] * fake.device_count + assert "nvmlDeviceGetMemoryInfo_v2" in fake.calls + assert "nvmlDeviceGetMemoryInfo" not in fake.calls + + +def test_detect_info_falls_back_to_v1_memory_when_v2_unavailable(fake_pyixml): + fake = fake_pyixml(v2_memory=False) + + devices = IluvatarDetector().detect_info() + + assert [dev.memory for dev in devices] == [ + byte_to_mebibyte(fake.v1_memory_total), + ] * fake.device_count + assert fake.calls.count("nvmlDeviceGetMemoryInfo_v2") == fake.device_count + assert fake.calls.count("nvmlDeviceGetMemoryInfo") == fake.device_count + + +def test_detect_info_has_no_vgpu_in_appendix(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert all("vgpu" not in dev.appendix for dev in devices) + + +def test_detect_info_keeps_the_minor_number_appendix(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect_info() + + assert [dev.appendix["minor_number"] for dev in devices] == [10, 11] + + +def test_detect_info_issues_no_usage_calls(fake_pyixml): + fake = fake_pyixml() + + IluvatarDetector().detect_info() + + usage_only_calls = { + "nvmlDeviceGetUtilizationRates", + "nvmlDeviceGetTemperature", + "nvmlDeviceGetPowerUsage", + } + assert usage_only_calls.isdisjoint(fake.calls) + + +# --------------------------------------------------------------------------- # +# detect_usage: merges by uuid, same V2->V1 fallback. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_the_usage_fields_by_uuid(fake_pyixml): + fake = fake_pyixml() + + devices = IluvatarDetector().detect_info() + fake.calls.clear() + + result = IluvatarDetector().detect_usage(devices) + + assert result is devices + for dev in devices: + assert dev.cores_utilization == 37 + assert dev.memory_used == byte_to_mebibyte(fake.v2_memory_used) + assert dev.temperature == 55 + assert dev.power_used == 90 + + +def test_detect_usage_falls_back_to_v1_memory_when_v2_unavailable(fake_pyixml): + fake = fake_pyixml(v2_memory=False) + + devices = IluvatarDetector().detect_info() + result = IluvatarDetector().detect_usage(devices) + + assert all( + dev.memory_used == byte_to_mebibyte(fake.v1_memory_used) for dev in result + ) + + +def test_detect_composes_info_and_usage_by_default(fake_pyixml): + fake_pyixml() + + devices = IluvatarDetector().detect() + + assert devices[0].cores_utilization == 37 + assert devices[0].power_used == 90 + assert "vgpu" not in devices[0].appendix + + +# --------------------------------------------------------------------------- # +# CDI: /dev/iluvatar{N} still reads the appendix minor number. # +# --------------------------------------------------------------------------- # + + +def test_cdi_reads_the_appendix_minor_number(monkeypatch): + seen_paths: list[str] = [] + + def _fake_device_node(path): + seen_paths.append(path) + return {"path": path} + + monkeypatch.setattr(cdi_iluvatar, "device_to_cdi_device_node", _fake_device_node) + + devices = [ + Device( + manufacturer=ManufacturerEnum.ILUVATAR, + index=0, + name="Iluvatar BI-V150", + uuid="GPU-0", + memory=32768, + appendix={"bdf": "0000:00:00.0", "minor_number": 7}, + ), + Device( + manufacturer=ManufacturerEnum.ILUVATAR, + index=1, + name="Iluvatar BI-V150", + uuid="GPU-1", + memory=32768, + appendix={"bdf": "0000:01:00.0"}, + ), + ] + + config = IluvatarGenerator().generate(devices) + + assert config is not None + # The device carrying a minor number is addressed by it. + assert "/dev/iluvatar7" in seen_paths + # The device without one falls back to Device.index. + assert "/dev/iluvatar1" in seen_paths diff --git a/tests/gpustack_runtime/detector/test_metax.py b/tests/gpustack_runtime/detector/test_metax.py index d86bb5c..6681aa3 100644 --- a/tests/gpustack_runtime/detector/test_metax.py +++ b/tests/gpustack_runtime/detector/test_metax.py @@ -1,5 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import metax, pymxsml +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.metax import MetaXDetector @@ -21,3 +31,360 @@ def test_get_topology(): det = MetaXDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pymxsml, so the split is provable on a host with no MetaX driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MetaX C500" + mode: int = pymxsml.MXSML_VIRTUALIZATION_MODE_NONE + bdf: str = "0000:01:00.0" + driver_version: str = "2.23.0.1" + vram_total: int = 67108864 # KiB, i.e. 65536 MiB + vram_used: int = 1048576 # KiB, i.e. 1024 MiB + dram_ue: int = 0 + core_usage: int = 42 + temperature: int = 5500 + power_limit: int = 350000 # mW, i.e. 350 W + power_ways: tuple[int, ...] = (120000, 30000) # mW, i.e. 150 W together + node_affinity: tuple[int, ...] = (0b1,) # NUMA node 0 + + +@dataclass +class _DeviceInfo: + """ + A c_mxsmlDeviceInfo_t stand-in, carrying the fields the detector reads. + """ + + uuid: str + deviceName: str # noqa: N815 + mode: int + bdfId: str # noqa: N815 + + +@dataclass +class _MemoryInfo: + """ + A c_mxsmlMemoryInfo_t stand-in, in KiB as the driver reports it. + """ + + vramTotal: int # noqa: N815 + vramUse: int # noqa: N815 + + +@dataclass +class _EccErrorCount: + """ + A c_mxSmlEccErrorCount_t stand-in, of which only the uncorrectable DRAM + errors decide the memory health. + """ + + dramUE: int # noqa: N815 + + +@dataclass +class _BoardWayElectricInfo: + """ + A c_mxSmlBoardWayElectricInfo_t stand-in, in mW as the driver reports it. + """ + + power: int + + +@dataclass +class _FakeMXSML: + """ + A stand-in for the pymxsml binding, recording every call the detector makes. + + The error type and the enumeration constants are the real module's, so a + fake drifting from the binding's contract fails here rather than on + hardware. Entry points are dispatched by name, as the fake libcndev.so of + test_pycndev.py does, which keeps the driver's camelCase out of the + handlers' own names. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + MXSMLError = pymxsml.MXSMLError + MXSML_VERSION_DRIVER = pymxsml.MXSML_VERSION_DRIVER + MXSML_USAGE_XCORE = pymxsml.MXSML_USAGE_XCORE + MXSML_TEMPERATURE_HOTSPOT = pymxsml.MXSML_TEMPERATURE_HOTSPOT + + def __getattr__(self, name: str): + handler = { + "mxSmlInit": self._init, + "mxSmlGetMacaVersion": self._get_maca_version, + "mxSmlGetDeviceCount": self._get_device_count, + "mxSmlGetDeviceVersion": self._get_device_version, + "mxSmlGetDeviceInfo": self._get_device_info, + "mxSmlGetMemoryInfo": self._get_memory_info, + "mxSmlGetTotalEccErrors": self._get_total_ecc_errors, + "mxSmlGetBoardPowerLimit": self._get_board_power_limit, + "mxSmlGetNodeAffinity": self._get_node_affinity, + "mxSmlGetDeviceIpUsage": self._get_device_ip_usage, + "mxSmlGetTemperatureInfo": self._get_temperature_info, + "mxSmlGetBoardPowerInfo": self._get_board_power_info, + }.get(name) + if handler is None: + msg = f"module pymxsml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _init(self) -> None: + pass + + def _get_maca_version(self) -> str: + return "2.33.0.6" + + def _get_device_count(self) -> int: + return len(self.cards) + + def _get_device_version(self, device_id: int, version_unit: int) -> str: + assert version_unit == pymxsml.MXSML_VERSION_DRIVER + return self.cards[device_id].driver_version + + def _get_device_info(self, device_id: int) -> _DeviceInfo: + card = self.cards[device_id] + return _DeviceInfo( + uuid=card.uuid, + deviceName=card.name, + mode=card.mode, + bdfId=card.bdf, + ) + + def _get_memory_info(self, device_id: int) -> _MemoryInfo: + card = self.cards[device_id] + return _MemoryInfo(vramTotal=card.vram_total, vramUse=card.vram_used) + + def _get_total_ecc_errors(self, device_id: int) -> _EccErrorCount: + return _EccErrorCount(dramUE=self.cards[device_id].dram_ue) + + def _get_board_power_limit(self, device_id: int) -> int: + return self.cards[device_id].power_limit + + def _get_node_affinity(self, device_id: int, node_set_size: int) -> list[int]: + assert node_set_size > 0 + return list(self.cards[device_id].node_affinity) + + def _get_device_ip_usage(self, device_id: int, usage_ip: int) -> int: + assert usage_ip == pymxsml.MXSML_USAGE_XCORE + return self.cards[device_id].core_usage + + def _get_temperature_info(self, device_id: int, temperature_type: int) -> int: + assert temperature_type == pymxsml.MXSML_TEMPERATURE_HOTSPOT + return self.cards[device_id].temperature + + def _get_board_power_info(self, device_id: int) -> list[_BoardWayElectricInfo]: + return [ + _BoardWayElectricInfo(power=power) + for power in self.cards[device_id].power_ways + ] + + +_USAGE_CALLS = ( + "mxSmlGetDeviceIpUsage", + "mxSmlGetTemperatureInfo", + "mxSmlGetBoardPowerInfo", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: mxSmlGetBoardPowerLimit +reads the power *limit* and mxSmlGetMemoryInfo the memory *total*, both of which +are inventory. +""" + +_MEMORY_UTILIZATION = 1.56 +""" +A default card's memory utilization: 1024 MiB used of 65536 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch): + """ + Build a MetaX detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card) -> tuple[MetaXDetector, _FakeMXSML]: + fake = _FakeMXSML(cards=list(cards)) + monkeypatch.setattr(metax, "pymxsml", fake) + det = MetaXDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: no virtualization-mode filter, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_every_virtualization_mode(detector): + det, _ = detector( + _Card(uuid="GPU-bare", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_NONE), + _Card(uuid="GPU-pf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_PF), + _Card(uuid="GPU-vf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_VF), + ) + + devices = det.detect_info() + + # The PF row is the regression guard: the detector used to `continue` on + # MXSML_VIRTUALIZATION_MODE_PF, i.e. drop the physical function -- the whole + # card -- so a virtualization-enabled host came up with zero devices. The VF + # row pins the other half: no virtualization-mode filter is applied at all, + # which is a deliberate divergence from the operator, as it still drops VF. + assert [dev.uuid for dev in devices] == ["GPU-bare", "GPU-pf", "GPU-vf"] + assert [dev.index for dev in devices] == [0, 1, 2] + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.METAX + assert dev.name == "MetaX C500" + assert dev.uuid == "GPU-0" + assert dev.driver_version == "2.23.0.1" + assert dev.runtime_version == "2.33" + assert dev.runtime_version_original == "2.33.0.6" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["numa"] == "0" + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="GPU-0")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + assert "mxSmlGetBoardPowerLimit" in fake.calls + assert "mxSmlGetMemoryInfo" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="GPU-pf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_PF), + _Card(uuid="GPU-vf", mode=pymxsml.MXSML_VIRTUALIZATION_MODE_VF), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="GPU-0"), + _Card( + uuid="GPU-1", + core_usage=7, + vram_used=2097152, + temperature=6100, + power_ways=(200000,), + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["GPU-0"].cores_utilization == 42 + assert by_uuid["GPU-0"].memory_used == 1024 + assert by_uuid["GPU-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["GPU-0"].temperature == 55 + assert by_uuid["GPU-0"].power_used == 150 + assert by_uuid["GPU-1"].cores_utilization == 7 + assert by_uuid["GPU-1"].memory_used == 2048 + assert by_uuid["GPU-1"].temperature == 61 + assert by_uuid["GPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["GPU-0"].index == 0 + assert by_uuid["GPU-0"].memory == 65536 + assert by_uuid["GPU-0"].power == 350 + assert by_uuid["GPU-0"].name == "MetaX C500" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].name == "MetaX C500" + assert devices[0].memory == 65536 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect()[0] + + assert dev.name == "MetaX C500" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reports_an_uncorrectable_ecc_error(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector(_Card(uuid="GPU-0", dram_ue=3)) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY diff --git a/tests/gpustack_runtime/detector/test_mig_devices.py b/tests/gpustack_runtime/detector/test_mig_devices.py index aec2131..02c9fd1 100644 --- a/tests/gpustack_runtime/detector/test_mig_devices.py +++ b/tests/gpustack_runtime/detector/test_mig_devices.py @@ -29,7 +29,7 @@ def _mig(slot: int, uuid: str) -> dict: "name": "1g.5gb", "uuid": uuid, "memory": 4864, - "appendix": {"vgpu": True, "sliced": True, "mig": True}, + "appendix": {"sliced": True, "mig": True}, } @@ -59,10 +59,10 @@ def test_index_mig_devices_keeps_the_blocks_apart(): assert len(indexes) == len(set(indexes)) -def test_index_mig_devices_clears_physical_indexes(): - # Physical indexes are minor numbers when physical index priority is on, - # so they are not bound by the card count: numbering from the count would - # collide with the cards themselves. +def test_index_mig_devices_offsets_from_the_reported_indexes(): + # A detector's reported index is not necessarily zero-based and contiguous + # -- Ascend reports the DCMI logic id -- so it is not bound by the card + # count: numbering from the count would collide with the cards themselves. cards = [_card(2, "GPU-2"), _card(3, "GPU-3")] mig_devices = { 0: [_mig(0, "MIG-2-0"), _mig(1, "MIG-2-1")], diff --git a/tests/gpustack_runtime/detector/test_mthreads.py b/tests/gpustack_runtime/detector/test_mthreads.py index 59170b6..45d51d7 100644 --- a/tests/gpustack_runtime/detector/test_mthreads.py +++ b/tests/gpustack_runtime/detector/test_mthreads.py @@ -1,5 +1,16 @@ +from __future__ import annotations + +import contextlib +from dataclasses import dataclass, field + import pytest +from gpustack_runtime import envs +from gpustack_runtime.detector import mthreads, pymtml +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) from gpustack_runtime.detector.mthreads import MThreadsDetector @@ -21,3 +32,414 @@ def test_get_topology(): det = MThreadsDetector() topo = det.get_topology() print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pymtml, so the split is provable on a host with no MThreads driver. # +# --------------------------------------------------------------------------- # + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "MTT S4000" + virt_role: int = pymtml.MTML_VIRT_ROLE_NONE + mpc_type: int = pymtml.MTML_MPC_TYPE_NONE + bus: int = 0x01 + cores: int = 128 + memory_total: int = 51539607552 # byte, i.e. 49152 MiB + memory_used: int = 1073741824 # byte, i.e. 1024 MiB + memory_ecc_dram_ue: int = 0 + cores_utilization: int = 42 + temperature: int = 55 + power_usage: int = 150 + node_affinity: tuple[int, ...] = (0b1,) # NUMA node 0 + + +@dataclass +class _DeviceProperty: + """ + A c_mtmlDeviceProperty_t stand-in. + + Its real field names, deliberately: the removed filter read a `mpcCap` + field that the struct does not have -- it carries `mpcCapability` and + `mpcType` -- so on a card reporting virtRole HOST_VIRTDEVICE the read raised + AttributeError and failed the whole detect pass. + """ + + virtCapability: int # noqa: N815 + virtRole: int # noqa: N815 + mpcCapability: int # noqa: N815 + mpcType: int # noqa: N815 + + +@dataclass +class _PciInfo: + """ + A c_mtmlPciInfo_t stand-in, carrying the fields the detector reads. + """ + + segment: int + bus: int + device: int + + +@dataclass +class _Handle: + """ + A device/memory/GPU handle stand-in, carrying the card it belongs to. + """ + + index: int + + +@dataclass +class _FakeMTML: + """ + A stand-in for the pymtml binding, recording every call the detector makes. + + The error type and the enumeration constants the detector legitimately needs + are the real module's, so a fake drifting from the binding's contract fails + here rather than on hardware. The virtRole and MPC constants are left out on + purpose: a reintroduced virtRole filter then fails loudly with AttributeError + instead of quietly reporting fewer cards. Entry points are dispatched by + name, which keeps the driver's camelCase out of the handlers' own names. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + + MTMLError = pymtml.MTMLError + MTML_MEMORY_ERROR_TYPE_UNCORRECTED = pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED + MTML_VOLATILE_ECC = pymtml.MTML_VOLATILE_ECC + MTML_MEMORY_LOCATION_DRAM = pymtml.MTML_MEMORY_LOCATION_DRAM + + def __getattr__(self, name: str): + handler = { + "mtmlLibraryInit": self._library_init, + "mtmlLibraryInitSystem": self._library_init_system, + "mtmlLibraryFreeSystem": self._library_free_system, + "mtmlLibraryCountDevice": self._library_count_device, + "mtmlLibraryInitDeviceByIndex": self._library_init_device_by_index, + "mtmlLibraryFreeDevice": self._library_free_device, + "mtmlSystemGetDriverVersion": self._system_get_driver_version, + "mtmlDeviceGetProperty": self._device_get_property, + "mtmlDeviceGetUUID": self._device_get_uuid, + "mtmlDeviceGetName": self._device_get_name, + "mtmlDeviceCountGpuCores": self._device_count_gpu_cores, + "mtmlDeviceGetPciInfo": self._device_get_pci_info, + "mtmlDeviceGetPowerUsage": self._device_get_power_usage, + "mtmlDeviceGetMemoryAffinityWithinNode": self._device_get_memory_affinity, + "mtmlMemoryContext": self._memory_context, + "mtmlMemoryGetTotal": self._memory_get_total, + "mtmlMemoryGetUsed": self._memory_get_used, + "mtmlMemoryGetEccErrorCounter": self._memory_get_ecc_error_counter, + "mtmlGpuContext": self._gpu_context, + "mtmlGpuGetUtilization": self._gpu_get_utilization, + "mtmlGpuGetTemperature": self._gpu_get_temperature, + }.get(name) + if handler is None: + msg = f"module pymtml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args): + self.calls.append(name) + return handler(*args) + + return entry_point + + def _library_init(self) -> None: + pass + + def _library_init_system(self) -> object: + return object() + + def _library_free_system(self, system) -> None: + pass + + def _library_count_device(self) -> int: + return len(self.cards) + + def _library_init_device_by_index(self, index: int) -> _Handle: + return _Handle(index=index) + + def _library_free_device(self, device: _Handle) -> None: + pass + + def _system_get_driver_version(self, system) -> str: + return "2.7.0" + + def _device_get_property(self, device: _Handle) -> _DeviceProperty: + card = self.cards[device.index] + return _DeviceProperty( + virtCapability=0, + virtRole=card.virt_role, + mpcCapability=0, + mpcType=card.mpc_type, + ) + + def _device_get_uuid(self, device: _Handle) -> str: + return self.cards[device.index].uuid + + def _device_get_name(self, device: _Handle) -> str: + return self.cards[device.index].name + + def _device_count_gpu_cores(self, device: _Handle) -> int: + return self.cards[device.index].cores + + def _device_get_pci_info(self, device: _Handle) -> _PciInfo: + return _PciInfo(segment=0, bus=self.cards[device.index].bus, device=0) + + def _device_get_power_usage(self, device: _Handle) -> int: + return self.cards[device.index].power_usage + + def _device_get_memory_affinity( + self, + device: _Handle, + node_set_size: int, + ) -> list[int]: + assert node_set_size > 0 + return list(self.cards[device.index].node_affinity) + + def _memory_context(self, device: _Handle): + return contextlib.nullcontext(_Handle(index=device.index)) + + def _memory_get_total(self, memory: _Handle) -> int: + return self.cards[memory.index].memory_total + + def _memory_get_used(self, memory: _Handle) -> int: + return self.cards[memory.index].memory_used + + def _memory_get_ecc_error_counter( + self, + memory: _Handle, + error_type: int, + counter_type: int, + location_type: int, + ) -> int: + assert error_type == pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED + assert counter_type == pymtml.MTML_VOLATILE_ECC + assert location_type == pymtml.MTML_MEMORY_LOCATION_DRAM + return self.cards[memory.index].memory_ecc_dram_ue + + def _gpu_context(self, device: _Handle): + return contextlib.nullcontext(_Handle(index=device.index)) + + def _gpu_get_utilization(self, gpu: _Handle) -> int: + return self.cards[gpu.index].cores_utilization + + def _gpu_get_temperature(self, gpu: _Handle) -> int: + return self.cards[gpu.index].temperature + + +_USAGE_CALLS = ( + "mtmlGpuContext", + "mtmlGpuGetUtilization", + "mtmlGpuGetTemperature", + "mtmlDeviceGetPowerUsage", +) +""" +The calls the usage query owns, i.e. the ones the information query must not +make. Deliberately not the whole metric-looking surface: mtmlMemoryGetTotal +reads the memory *total*, which is inventory. MTML binds no power *limit* call +at all, so nothing of the sort stays behind in the information query. +""" + +_MEMORY_UTILIZATION = 2.08 +""" +A default card's memory utilization: 1024 MiB used of 49152 MiB total, as +get_utilization rounds it. +""" + + +@pytest.fixture +def detector(monkeypatch): + """ + Build an MThreads detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card) -> tuple[MThreadsDetector, _FakeMTML]: + fake = _FakeMTML(cards=list(cards)) + monkeypatch.setattr(mthreads, "pymtml", fake) + det = MThreadsDetector() + # Shadowed on the instance, so the lru_cache'd static stays untouched. + monkeypatch.setattr(det, "is_supported", lambda: True) + return det, fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: no virtRole filter, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_every_virt_role(detector): + det, _ = detector( + _Card(uuid="GPU-bare", virt_role=pymtml.MTML_VIRT_ROLE_NONE), + _Card( + uuid="GPU-host-virt", + virt_role=pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE, + mpc_type=pymtml.MTML_MPC_TYPE_PARENT, # i.e. != MPC_TYPE_INSTANCE + ), + _Card( + uuid="GPU-guest-virt", + virt_role=pymtml.MTML_VIRT_ROLE_GUEST_VIRTDEVICE, + ), + ) + + devices = det.detect_info() + + # The host-virt row is the regression guard: the detector used to read a + # `mpcCap` field that c_mtmlDeviceProperty_t does not carry, so such a card + # raised AttributeError and failed the whole detect pass -- the worker saw + # zero MThreads devices, not merely a missing one. The guest-virt row pins + # the other half: no virtRole filter is applied at all, which is a deliberate + # divergence from the operator, as it still drops GUEST_VIRTDEVICE. + assert [dev.uuid for dev in devices] == [ + "GPU-bare", + "GPU-host-virt", + "GPU-guest-virt", + ] + assert [dev.index for dev in devices] == [0, 1, 2] + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.MTHREADS + assert dev.name == "MTT S4000" + assert dev.uuid == "GPU-0" + assert dev.driver_version == "2.7.0" + assert dev.cores == 128 + assert dev.memory == 49152 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["numa"] == "0" + # MTML binds no power *limit* call, so the field stays unset rather than + # borrowing the power usage the usage query reads. + assert dev.power is None + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector(_Card(uuid="GPU-0")) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # The property read went away with the filter it served, and nothing else + # consumes it. + assert "mtmlDeviceGetProperty" not in fake.calls + assert "mtmlMemoryGetTotal" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="GPU-host-virt", virt_role=pymtml.MTML_VIRT_ROLE_HOST_VIRTDEVICE), + _Card(uuid="GPU-guest-virt", virt_role=pymtml.MTML_VIRT_ROLE_GUEST_VIRTDEVICE), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="GPU-0"), + _Card( + uuid="GPU-1", + cores_utilization=7, + memory_used=2147483648, # byte, i.e. 2048 MiB + temperature=61, + power_usage=200, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["GPU-0"].cores_utilization == 42 + assert by_uuid["GPU-0"].memory_used == 1024 + assert by_uuid["GPU-0"].memory_utilization == _MEMORY_UTILIZATION + assert by_uuid["GPU-0"].temperature == 55 + assert by_uuid["GPU-0"].power_used == 150 + assert by_uuid["GPU-1"].cores_utilization == 7 + assert by_uuid["GPU-1"].memory_used == 2048 + assert by_uuid["GPU-1"].temperature == 61 + assert by_uuid["GPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["GPU-0"].index == 0 + assert by_uuid["GPU-0"].memory == 49152 + assert by_uuid["GPU-0"].cores == 128 + assert by_uuid["GPU-0"].name == "MTT S4000" + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["GPU-0"] + assert devices[0].name == "MTT S4000" + assert devices[0].memory == 49152 + assert devices[0].cores_utilization == 42 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + dev = det.detect()[0] + + assert dev.name == "MTT S4000" + assert dev.memory == 49152 + assert dev.cores_utilization == 42 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector(_Card(uuid="GPU-0")) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_reports_an_uncorrectable_ecc_error(detector, monkeypatch): + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + det, _ = detector(_Card(uuid="GPU-0", memory_ecc_dram_ue=3)) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + assert det.detect_info()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert det.detect()[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY diff --git a/tests/gpustack_runtime/detector/test_nvidia.py b/tests/gpustack_runtime/detector/test_nvidia.py index b128212..1562e34 100644 --- a/tests/gpustack_runtime/detector/test_nvidia.py +++ b/tests/gpustack_runtime/detector/test_nvidia.py @@ -1,7 +1,755 @@ +from __future__ import annotations + +import ctypes +import dataclasses +from types import SimpleNamespace + import pytest +from gpustack_runtime.detector import nvidia +from gpustack_runtime.detector.__types__ import DeviceMemoryStatusEnum +from gpustack_runtime.detector.__utils__ import get_utilization from gpustack_runtime.detector.nvidia import NVIDIADetector +# --------------------------------------------------------------------------- # +# A fake pynvml, built in this module rather than shared: every vendor binding # +# exposes an unrelated API, and this one carries a call log because several # +# criteria are "issues no metric call", which no return value can prove. # +# --------------------------------------------------------------------------- # + +_MIB = 1 << 20 + +_USAGE_CALLS = ( + "nvmlGpmQueryDeviceSupport", + "nvmlGpmSampleGet", + "nvmlGpmMigSampleGet", + "nvmlGpmMetricsGet", + "nvmlDeviceGetUtilizationRates", + "nvmlDeviceGetTemperature", + "nvmlDeviceGetPowerUsage", +) +""" +The driver calls only the usage query is allowed to make. +""" + + +class _NVMLError(Exception): + """ + The fake binding's error type, standing in for pynvml.NVMLError. + """ + + +class _FakeFabricInfo(ctypes.Structure): + """ + A byref()-able fabric info: the detector takes this struct's address. + """ + + _fields_ = ( + ("state", ctypes.c_uint), + ("clusterUuid", ctypes.c_ubyte * 16), + ("cliqueId", ctypes.c_uint), + ) + + +class _FakeGpmMetrics: + """ + A GPM metrics request, as the detector fills and reads it. + """ + + def __init__(self): + self.version = 0 + self.numMetrics = 0 + self.sample1 = None + self.sample2 = None + self.metrics = [SimpleNamespace(metricId=0, value=float("nan"))] + + +@dataclasses.dataclass +class _FakeMigDevice: + """ + A MIG device of a MIG-enabled card. + """ + + uuid: str + gpu_instance_id: int = 0 + compute_instance_id: int = 0 + memory: int = 4864 * _MIB + memory_used: int = 512 * _MIB + ecc_errors: int = 0 + sm_util: float | None = None + mig_devices: None = None + + +@dataclasses.dataclass +class _FakeDevice: + """ + A card, as the fake binding reports it. Every field is a knob a test turns. + """ + + uuid: str = "GPU-0" + name: str = "NVIDIA L4" + minor_number: int | None = 0 + bdf: str = "0000:6A:00.0" + compute_capability: tuple[int, int] = (8, 9) + cores: int = 7424 + memory: int = 23034 * _MIB + memory_used: int = 1024 * _MIB + memory_bus_width: int | None = 192 + ecc_mode: int = 1 # NVML_FEATURE_ENABLED + ecc_errors: int = 0 + temperature: int = 47 + power_limit: int = 72_000 # mW + power_used: int = 30_000 # mW + cores_utilization: int = 12 + sm_util: float | None = None + """ + The SM utilization GPM samples, or None for a card not supporting GPM. + """ + mig_devices: list[_FakeMigDevice] | None = None + """ + The card's MIG devices, or None for a card with MIG disabled. + """ + + +class _FakeNVML: + """ + A pynvml stand-in exposing only what the NVIDIA detector touches, recording + every call it receives. + """ + + NVMLError = _NVMLError + NVML_SUCCESS = 0 + NVML_ERROR_NOT_SUPPORTED = 3 + NVML_FEATURE_DISABLED = 0 + NVML_FEATURE_ENABLED = 1 + NVML_TEMPERATURE_GPU = 0 + NVML_AFFINITY_SCOPE_NODE = 1 + NVML_DEVICE_MIG_DISABLE = 0 + NVML_DEVICE_MIG_ENABLE = 1 + NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1 + NVML_VOLATILE_ECC = 0 + NVML_AGGREGATE_ECC = 1 + NVML_MEMORY_LOCATION_DRAM = 2 + NVML_MEMORY_LOCATION_SRAM = 5 + NVML_GPU_INSTANCE_PROFILE_COUNT = 3 + NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 2 + NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 1 + NVML_GPM_METRIC_SM_UTIL = 2 + NVML_GPM_METRICS_GET_VERSION = 1 + NVML_GPU_FABRIC_STATE_COMPLETED = 3 + + c_nvmlGpuFabricInfoV_t = _FakeFabricInfo # noqa: N815 + c_nvmlGpmMetricsGet_t = _FakeGpmMetrics # noqa: N815 + + mig_profile_name = "MIG 1g.5gb" + mig_profile_memory_mb = 4864 + mig_profile_cores = 14 + + def __init__( + self, + devices: list[_FakeDevice], + memory_v2_binding: bool = True, + memory_v2_driver: bool = True, + ): + """ + Args: + devices: + The cards the fake driver enumerates. + memory_v2_binding: + Whether the binding exposes the packed v2 struct version, as an + older nvidia-ml-py does not. + memory_v2_driver: + Whether the driver answers the v2 memory call. + + """ + self.devices = list(devices) + self.calls: list[str] = [] + self.memory_versions: list[int | None] = [] + self.memory_v2_driver = memory_v2_driver + if memory_v2_binding: + self.nvmlMemory_v2 = 33554472 + self._gpm_target: _FakeDevice | _FakeMigDevice | None = None + + # System. + + def nvmlInit(self): # noqa: N802 + self.calls.append("nvmlInit") + + def nvmlSystemGetDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetDriverVersion") + return "580.65.06" + + def nvmlSystemGetCudaDriverVersion(self): # noqa: N802 + self.calls.append("nvmlSystemGetCudaDriverVersion") + return 13000 + + def nvmlDeviceGetCount(self): # noqa: N802 + self.calls.append("nvmlDeviceGetCount") + return len(self.devices) + + def nvmlDeviceGetHandleByIndex(self, index): # noqa: N802 + self.calls.append("nvmlDeviceGetHandleByIndex") + return self.devices[index] + + # Identity and capability. + + def nvmlDeviceGetUUID(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetUUID") + return handle.uuid + + def nvmlDeviceGetName(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetName") + return handle.name + + def nvmlDeviceGetCudaComputeCapability(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetCudaComputeCapability") + return list(handle.compute_capability) + + def nvmlDeviceGetPciInfo(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPciInfo") + # The legacy tuple is uppercase hexadecimal, as NVML formats it. + return SimpleNamespace(busIdLegacy=handle.bdf) + + def nvmlDeviceGetMemoryAffinity(self, handle, size, scope): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryAffinity") + msg = "no memory affinity" + raise _NVMLError(msg) + + def nvmlDeviceGetNumGpuCores(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetNumGpuCores") + return handle.cores + + def nvmlDeviceGetMinorNumber(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMinorNumber") + if handle.minor_number is None: + msg = "no minor number" + raise _NVMLError(msg) + return handle.minor_number + + def nvmlDeviceGetPowerManagementDefaultLimit(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerManagementDefaultLimit") + return handle.power_limit + + def nvmlDeviceGetGpuFabricInfoV(self, handle, info_ref): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuFabricInfoV") + return self.NVML_ERROR_NOT_SUPPORTED + + # Memory. + + def nvmlDeviceGetMemoryInfo(self, handle, version=None): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryInfo") + self.memory_versions.append(version) + if version is not None and not self.memory_v2_driver: + msg = "no v2 memory info" + raise _NVMLError(msg) + return SimpleNamespace( + total=handle.memory, + used=handle.memory_used, + free=handle.memory - handle.memory_used, + ) + + def nvmlDeviceGetMemoryBusWidth(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryBusWidth") + if handle.memory_bus_width is None: + msg = "no memory bus width" + raise _NVMLError(msg) + return handle.memory_bus_width + + def nvmlDeviceGetEccMode(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetEccMode") + return [handle.ecc_mode, handle.ecc_mode] + + def nvmlDeviceGetMemoryErrorCounter(self, handle, error_type, scope, location): # noqa: N802 + self.calls.append("nvmlDeviceGetMemoryErrorCounter") + return handle.ecc_errors + + # Usage. + + def nvmlDeviceGetUtilizationRates(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetUtilizationRates") + return SimpleNamespace(gpu=handle.cores_utilization, memory=0) + + def nvmlDeviceGetTemperature(self, handle, sensor): # noqa: N802 + self.calls.append("nvmlDeviceGetTemperature") + return handle.temperature + + def nvmlDeviceGetPowerUsage(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetPowerUsage") + return handle.power_used + + def nvmlGpmQueryDeviceSupport(self, handle): # noqa: N802 + self.calls.append("nvmlGpmQueryDeviceSupport") + return SimpleNamespace(isSupportedDevice=int(handle.sm_util is not None)) + + def nvmlGpmSampleAlloc(self): # noqa: N802 + self.calls.append("nvmlGpmSampleAlloc") + return object() + + def nvmlGpmSampleFree(self, sample): # noqa: N802 + self.calls.append("nvmlGpmSampleFree") + + def nvmlGpmSampleGet(self, handle, sample): # noqa: N802 + self.calls.append("nvmlGpmSampleGet") + self._gpm_target = handle + + def nvmlGpmMigSampleGet(self, handle, gpu_instance_id, sample): # noqa: N802 + self.calls.append("nvmlGpmMigSampleGet") + self._gpm_target = next( + ( + mig + for mig in handle.mig_devices or [] + if mig.gpu_instance_id == gpu_instance_id + ), + None, + ) + + def nvmlGpmMetricsGet(self, metrics): # noqa: N802 + self.calls.append("nvmlGpmMetricsGet") + sm_util = getattr(self._gpm_target, "sm_util", None) + metrics.metrics[0].value = float("nan") if sm_util is None else float(sm_util) + + # MIG. + + def nvmlDeviceGetMigMode(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMigMode") + mode = ( + self.NVML_DEVICE_MIG_DISABLE + if handle.mig_devices is None + else self.NVML_DEVICE_MIG_ENABLE + ) + return mode, mode + + def nvmlDeviceGetMaxMigDeviceCount(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetMaxMigDeviceCount") + return 7 + + def nvmlDeviceGetMigDeviceHandleByIndex(self, handle, index): # noqa: N802 + self.calls.append("nvmlDeviceGetMigDeviceHandleByIndex") + migs = handle.mig_devices or [] + if index >= len(migs): + msg = "no MIG device at that slot" + raise _NVMLError(msg) + return migs[index] + + def nvmlDeviceGetGpuInstanceId(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceId") + return handle.gpu_instance_id + + def nvmlDeviceGetComputeInstanceId(self, handle): # noqa: N802 + self.calls.append("nvmlDeviceGetComputeInstanceId") + return handle.compute_instance_id + + def nvmlDeviceGetGpuInstanceById(self, handle, gpu_instance_id): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceById") + return SimpleNamespace(card=handle, gpu_instance_id=gpu_instance_id) + + def nvmlGpuInstanceGetComputeInstanceById(self, gpu_instance, compute_instance_id): # noqa: N802 + self.calls.append("nvmlGpuInstanceGetComputeInstanceById") + return SimpleNamespace(compute_instance_id=compute_instance_id) + + def nvmlGpuInstanceGetInfo(self, gpu_instance): # noqa: N802 + self.calls.append("nvmlGpuInstanceGetInfo") + return SimpleNamespace(profileId=0) + + def nvmlComputeInstanceGetInfo(self, compute_instance): # noqa: N802 + self.calls.append("nvmlComputeInstanceGetInfo") + return SimpleNamespace(profileId=0) + + def nvmlDeviceGetGpuInstanceProfileInfo(self, handle, profile_id): # noqa: N802 + self.calls.append("nvmlDeviceGetGpuInstanceProfileInfo") + if profile_id != 0: + msg = "no such GPU instance profile" + raise _NVMLError(msg) + return SimpleNamespace( + id=0, + memorySizeMB=self.mig_profile_memory_mb, + sliceCount=1, + name=self.mig_profile_name, + ) + + def nvmlGpuInstanceGetComputeInstanceProfileInfo( # noqa: N802 + self, + gpu_instance, + profile_id, + engine_profile_id, + ): + self.calls.append("nvmlGpuInstanceGetComputeInstanceProfileInfo") + if (profile_id, engine_profile_id) != (0, 0): + msg = "no such compute instance profile" + raise _NVMLError(msg) + return SimpleNamespace(id=0, multiprocessorCount=self.mig_profile_cores) + + +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the ECC error check on: it is opt-in, as reading the counters costs a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults + to true. A real module attribute is set because the env lookup is cached. + """ + monkeypatch.setattr( + nvidia.envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + +@pytest.fixture +def fake_nvml(monkeypatch): + """ + Install a fake binding in place of pynvml, and return the installer. + """ + + def _install(devices: list[_FakeDevice], **kwargs) -> _FakeNVML: + fake = _FakeNVML(devices, **kwargs) + monkeypatch.setattr(nvidia, "pynvml", fake) + # is_supported() initializes the real driver, which the fake replaces. + monkeypatch.setattr(NVIDIADetector, "is_supported", staticmethod(lambda: True)) + # The NUMA node comes from sysfs, so it is answered here instead of + # letting the host decide what the test sees. + monkeypatch.setattr(nvidia, "get_numa_node_by_bdf", lambda *_: "") + # GPM samples over a 100 ms window of real time, twice per query. + monkeypatch.setattr(nvidia.time, "sleep", lambda *_: None) + return fake + + return _install + + +# --------------------------------------------------------------------------- # +# The information query. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_reports_the_card(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect_info() + + assert len(devices) == 1 + device = devices[0] + assert device.index == 0 + assert device.name == "NVIDIA L4" + assert device.uuid == "GPU-0" + assert device.driver_version == "580.65.06" + assert device.runtime_version == "13.0" + assert device.runtime_version_original == "13.0.0" + assert device.compute_capability == "8.9" + assert device.cores == 7424 + assert device.power == 72 + assert device.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert device.appendix == { + "arch_family": "Ada-Lovelace", + "mig": False, + "bdf": "0000:6a:00.0", + "minor_number": 0, + } + # The usage fields stay at their defaults: this query does not read them. + assert device.cores_utilization == 0 + assert device.memory_used == 0 + assert device.memory_utilization == 0 + assert device.temperature is None + assert device.power_used is None + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + + +def test_detect_info_issues_no_usage_call(fake_nvml): + # A card supporting GPM and hosting MIG devices, i.e. every usage call the + # detector knows is reachable. + fake = fake_nvml( + [ + _FakeDevice( + uuid="GPU-0", + name="NVIDIA H100 80GB HBM3", + memory_bus_width=5120, + sm_util=61.0, + mig_devices=[_FakeMigDevice(uuid="MIG-0-0", sm_util=33.0)], + ), + ], + ) + + NVIDIADetector().detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # ... while the inventory calls did happen. + assert "nvmlDeviceGetMemoryInfo" in fake.calls + assert "nvmlDeviceGetPowerManagementDefaultLimit" in fake.calls + assert "nvmlDeviceGetMigDeviceHandleByIndex" in fake.calls + + +def test_detect_info_records_no_vgpu(fake_nvml): + # Whole-card reporting: no virtual/PF/VF classification anywhere. + fake_nvml([_FakeDevice(mig_devices=[_FakeMigDevice(uuid="MIG-0-0")])]) + + devices = NVIDIADetector().detect_info() + + assert "vgpu" not in devices[0].appendix + assert "vgpu" not in devices[0].appendix["mig_devices"][0]["appendix"] + assert not hasattr(nvidia, "_is_vgpu") + + +def test_detect_info_omits_an_unreadable_minor_number(fake_nvml): + fake_nvml([_FakeDevice(minor_number=None)]) + + devices = NVIDIADetector().detect_info() + + assert "minor_number" not in devices[0].appendix + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_reports_the_memory_health(fake_nvml): + fake_nvml([_FakeDevice(ecc_errors=1)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# Memory is what the card can allocate, not its ECC-restored capacity. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "bus_width, ecc_mode", + [ + (192, 1), # L4: GDDR with ECC on -- where the operator would restore. + (384, 1), # L40S: the widest GDDR bus. + (5120, 1), # H100: HBM keeps ECC out of the user-visible memory anyway. + (384, 0), # GDDR, ECC off: nothing was carved out. + (None, 1), # An unreadable bus width. + ], +) +def test_detect_info_reports_the_allocatable_memory(fake_nvml, bus_width, ecc_mode): + # A deliberate divergence from the operator, which adds back the ~1/16 that + # ECC parity carves out of a GDDR part. That capacity is not reachable, and + # the operator's restored figure is a display value taking no part in + # allocation -- but `memory` here does, and `memory - memory_used` has to + # mean free space, so restoring it would over-commit the card. + total = 23034 + fake_nvml( + [ + _FakeDevice( + memory=total * _MIB, + memory_bus_width=bus_width, + ecc_mode=ecc_mode, + ), + ], + ) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory == total + + +def test_detect_falls_back_to_the_host_memory(fake_nvml, monkeypatch): + # A deliberate divergence from the operator, which skips such a device. + fake_nvml([_FakeDevice(memory=0, memory_used=0)]) + monkeypatch.setattr(nvidia, "get_memory", lambda: (65536, 4096)) + + detector = NVIDIADetector() + devices = detector.detect_info() + + assert devices[0].memory == 65536 + assert devices[0].memory_used == 0 + + detector.detect_usage(devices) + + assert devices[0].memory_used == 4096 + assert devices[0].memory_utilization == get_utilization(4096, 65536) + + +# --------------------------------------------------------------------------- # +# The v2 memory structure. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_prefers_the_v2_memory_structure(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + NVIDIADetector().detect_info() + + # The packed struct version the driver validates, not a plain ordinal. + assert fake.memory_versions == [fake.nvmlMemory_v2] + + +def test_detect_info_falls_back_to_the_v1_memory_structure(fake_nvml): + fake = fake_nvml([_FakeDevice()], memory_v2_driver=False) + + devices = NVIDIADetector().detect_info() + + assert fake.memory_versions == [fake.nvmlMemory_v2, None] + assert devices[0].memory == 23034 + + +def test_detect_info_skips_the_v2_call_the_binding_lacks(fake_nvml): + fake = fake_nvml([_FakeDevice()], memory_v2_binding=False) + + devices = NVIDIADetector().detect_info() + + assert fake.memory_versions == [None] + assert devices[0].memory == 23034 + + +# --------------------------------------------------------------------------- # +# The usage query. # +# --------------------------------------------------------------------------- # + + +def test_detect_fills_the_usage_fields(fake_nvml): + fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect() + + device = devices[0] + assert device.cores_utilization == 12 + assert device.memory_used == 1024 + # The three memory fields of one card agree: the utilization is measured + # against the same total that `memory` reports. + assert device.memory == 23034 + assert device.memory_utilization == get_utilization(1024, 23034) + assert device.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert device.temperature == 47 + assert device.power_used == 30 + + +def test_detect_without_usage_leaves_the_usage_fields_alone(fake_nvml): + fake = fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect(usage=False) + + assert devices[0].cores_utilization == 0 + assert devices[0].temperature is None + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + + +def test_detect_usage_merges_into_the_given_devices(fake_nvml): + fake_nvml([_FakeDevice(uuid="GPU-0"), _FakeDevice(uuid="GPU-1")]) + detector = NVIDIADetector() + devices = detector.detect_info() + + merged = detector.detect_usage(devices) + + assert merged is devices + assert [device.temperature for device in devices] == [47, 47] + # The information fields survive the merge. + assert [device.uuid for device in devices] == ["GPU-0", "GPU-1"] + assert [device.name for device in devices] == ["NVIDIA L4"] * 2 + assert [device.memory for device in devices] == [23034] * 2 + + +def test_detect_usage_detects_the_information_first(fake_nvml): + fake_nvml([_FakeDevice()]) + + devices = NVIDIADetector().detect_usage() + + assert [device.uuid for device in devices] == ["GPU-0"] + assert devices[0].cores_utilization == 12 + + +def test_detect_usage_prefers_gpm(fake_nvml): + fake = fake_nvml([_FakeDevice(sm_util=61.4)]) + + devices = NVIDIADetector().detect() + + assert devices[0].cores_utilization == 61 + assert "nvmlDeviceGetUtilizationRates" not in fake.calls + + +@pytest.mark.usefixtures("health_check") +def test_detect_usage_reports_the_memory_health(fake_nvml): + fake_nvml([_FakeDevice(ecc_errors=1)]) + + devices = NVIDIADetector().detect() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +# --------------------------------------------------------------------------- # +# The MIG devices, which stay in the card's appendix. # +# --------------------------------------------------------------------------- # + + +def _mig_card(**kwargs) -> _FakeDevice: + return _FakeDevice( + uuid="GPU-0", + name="NVIDIA A100-SXM4-40GB", + compute_capability=(8, 0), + memory=40960 * _MIB, + memory_bus_width=5120, + # GPM is queried on the card, whether the sample is the card's or an + # instance's. + sm_util=55.0, + mig_devices=[ + _FakeMigDevice(uuid="MIG-0-0", gpu_instance_id=1, sm_util=33.0), + _FakeMigDevice(uuid="MIG-0-1", gpu_instance_id=2, sm_util=44.0), + ], + **kwargs, + ) + + +def test_detect_info_keeps_the_mig_devices_in_the_appendix(fake_nvml): + fake_nvml([_mig_card()]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].appendix["mig"] is True + mig_devices = devices[0].appendix["mig_devices"] + assert [mig["uuid"] for mig in mig_devices] == ["MIG-0-0", "MIG-0-1"] + assert [mig["name"] for mig in mig_devices] == ["1g.5gb"] * 2 + # Numbered by index_mig_devices: a block above the cards' own indexes. + assert [mig["index"] for mig in mig_devices] == [1, 2] + assert [mig["memory"] for mig in mig_devices] == [4864] * 2 + assert [mig["cores"] for mig in mig_devices] == [14] * 2 + assert mig_devices[0]["appendix"]["sliced"] is True + assert mig_devices[0]["appendix"]["mig"] is True + assert mig_devices[0]["appendix"]["gpu_instance_id"] == 1 + # The usage fields of an instance stay at their defaults too. + assert mig_devices[0]["cores_utilization"] == 0 + assert mig_devices[0]["memory_used"] == 0 + assert mig_devices[0]["memory_utilization"] == 0 + assert mig_devices[0]["temperature"] is None + assert mig_devices[0]["power_used"] is None + + +def test_detect_fills_the_mig_devices_usage(fake_nvml): + fake_nvml([_mig_card()]) + + devices = NVIDIADetector().detect() + + mig_devices = devices[0].appendix["mig_devices"] + assert [mig["cores_utilization"] for mig in mig_devices] == [33, 44] + assert [mig["memory_used"] for mig in mig_devices] == [512] * 2 + assert [mig["memory_utilization"] for mig in mig_devices] == [ + get_utilization(512, 4864), + ] * 2 + assert [mig["memory_status"] for mig in mig_devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + ] * 2 + # An instance reports neither temperature nor power, so it carries the + # card's, as the information query's entries do. + assert [mig["temperature"] for mig in mig_devices] == [47] * 2 + assert [mig["power_used"] for mig in mig_devices] == [30] * 2 + # The instances' own information fields are untouched by the merge. + assert [mig["index"] for mig in mig_devices] == [1, 2] + assert [mig["memory"] for mig in mig_devices] == [4864] * 2 + + +def test_detect_reports_a_mig_enabled_card_without_instances(fake_nvml): + # MIG on, nothing partitioned yet: the card is still the reported device. + fake_nvml([_FakeDevice(mig_devices=[])]) + + devices = NVIDIADetector().detect() + + assert len(devices) == 1 + assert devices[0].appendix["mig"] is True + assert devices[0].appendix["mig_devices"] == [] + + +# --------------------------------------------------------------------------- # +# The hardware paths, exercised only where a driver exists. # +# --------------------------------------------------------------------------- # + @pytest.mark.skipif( not NVIDIADetector.is_supported(), diff --git a/tests/gpustack_runtime/detector/test_pycndev.py b/tests/gpustack_runtime/detector/test_pycndev.py new file mode 100644 index 0000000..af8922d --- /dev/null +++ b/tests/gpustack_runtime/detector/test_pycndev.py @@ -0,0 +1,926 @@ +from __future__ import annotations as __future_annotations__ + +import importlib.util +import sys +import threading +from ctypes import sizeof +from unittest import mock + +import pytest + +from gpustack_runtime.detector import pycndev + +MODULE_NAME = "gpustack_runtime.detector.pycndev" + +# Expected layout of every bound struct: total size, then (field, offset, size) in the +# header's own field order. +# +# Derived from the operator's binding/cndev/cndev.h -- the header of record -- by +# compiling a probe that prints sizeof() and offsetof() for each field, so the table is +# independent of the ctypes definitions it checks. That independence is the point: no +# Cambricon hardware exists in this suite, the driver fills these buffers by offset, and +# a wrong width or a missing field would corrupt memory rather than fail a test. +# +# Every struct here is plain fixed-width scalars and arrays -- no bitfields, no packing, +# no pointers -- so these numbers hold on Linux x86_64 and arm64 alike. They are not +# specific to the host that ran the probe. +EXPECTED_LAYOUTS = { + "c_cndevCardInfo_t": ( + 8, + [("version", 0, 4), ("number", 4, 4)], + ), + "c_cndevUUID_t": ( + # uuid is 37 bytes at offset 4, so 7 bytes of padding precede the 8-aligned + # ncsUUID64 at 48. + 56, + [("version", 0, 4), ("uuid", 4, 37), ("ncsUUID64", 48, 8)], + ), + "c_cndevMemoryInfoV2_t": ( + 176, + [ + ("physicalMemoryTotal", 0, 8), + ("physicalMemoryUsed", 8, 8), + ("reservedMemory", 16, 8), + ("virtualMemoryTotal", 24, 8), + ("virtualMemoryUsed", 32, 8), + ("globalMemory", 40, 8), + ("reserved", 48, 128), + ], + ), + "c_cndevVersionInfo_t": ( + 28, + [ + ("version", 0, 4), + ("mcuMajorVersion", 4, 4), + ("mcuMinorVersion", 8, 4), + ("mcuBuildVersion", 12, 4), + ("driverMajorVersion", 16, 4), + ("driverMinorVersion", 20, 4), + ("driverBuildVersion", 24, 4), + ], + ), + "c_cndevECCInfo_t": ( + # 4 bytes of padding follow the version, so the first counter is 8-aligned. + 72, + [ + ("version", 0, 4), + ("oneBitError", 8, 8), + ("multipleOneError", 16, 8), + ("multipleError", 24, 8), + ("multipleMultipleError", 32, 8), + ("correctedError", 40, 8), + ("uncorrectedError", 48, 8), + ("totalError", 56, 8), + ("addressForbiddenError", 64, 8), + ], + ), + "c_cndevDevicePowerInfo_t": ( + 84, + [ + ("usage", 0, 4), + ("cap", 4, 4), + ("machine", 8, 4), + ("tdp", 12, 4), + ("maxPower", 16, 4), + ("reserved", 20, 64), + ], + ), + "c_cndevTemperatureInfo_t": ( + 148, + [ + ("version", 0, 4), + ("board", 4, 4), + ("cluster", 8, 80), + ("memoryDie", 88, 32), + ("chip", 120, 4), + ("airInlet", 124, 4), + ("airOutlet", 128, 4), + ("memory", 132, 4), + ("videoInput", 136, 4), + ("cpu", 140, 4), + ("isp", 144, 4), + ], + ), + "c_cndevUtilizationInfo_t": ( + 328, + [ + ("version", 0, 4), + ("averageCoreUtilization", 4, 4), + ("coreUtilization", 8, 320), + ], + ), + "c_cndevCardName_t": ( + 8, + [("version", 0, 4), ("id", 4, 4)], + ), + "c_cndevPCIeInfoV2_t": ( + 64, + [ + ("subsystemId", 0, 4), + ("deviceId", 4, 4), + ("vendor", 8, 2), + ("subsystemVendor", 10, 2), + ("domain", 12, 4), + ("bus", 16, 4), + ("device", 20, 4), + ("function", 24, 4), + ("moduleId", 28, 2), + ("slotId", 30, 2), + ("reserved", 32, 32), + ], + ), + "c_cndevCardHealthState_t": ( + 16, + [ + ("version", 0, 4), + ("health", 4, 4), + ("deviceState", 8, 4), + ("driverState", 12, 4), + ], + ), + "c_cndevDiagErrorDetail_t": ( + 532, + [ + ("msg", 0, 512), + ("device_id", 512, 4), + ("bdf", 516, 4), + ("code", 520, 4), + ("category", 524, 4), + ("severity", 528, 4), + ], + ), + "c_cndevIncidentInfo_t": ( + 540, + [("system", 0, 4), ("health", 4, 4), ("error", 8, 532)], + ), + "c_cndevCardHealthStateV2_t": ( + # 64 incident slots, which the driver fills in place: a short struct here is a + # buffer overrun, not a missing field. + 34612, + [ + ("health", 0, 4), + ("deviceState", 4, 4), + ("driverState", 8, 4), + ("overallHealth", 12, 4), + ("incident_count", 16, 4), + ("incidents", 20, 34560), + ("reserved", 34580, 32), + ], + ), + "c_cndevNUMANodeId_t": ( + 8, + [("version", 0, 4), ("nodeId", 4, 4)], + ), +} + +# Constants pinned to the literals in the operator's binding/cndev/cndev.h -- the header +# of record -- rather than to the module's own definitions. +# +# Asserting a constant against itself proves nothing: CNDEV_VERSION_6 defined as 7 would +# leave this suite green while every versioned call on an MLU host was answered with +# CNDEV_ERROR_UNSUPPORTED_API_VERSION. No Cambricon hardware exists here, so a mistyped +# constant has to fail below or it fails at a customer. +HEADER_API_VERSION = 6 + +# cndevRet_enum, in the header's own order. +HEADER_ERROR_CODES = { + "CNDEV_SUCCESS": 0, + "CNDEV_ERROR_NO_DRIVER": 1, + "CNDEV_ERROR_LOW_DRIVER_VERSION": 2, + "CNDEV_ERROR_UNSUPPORTED_API_VERSION": 3, + "CNDEV_ERROR_UNINITIALIZED": 4, + "CNDEV_ERROR_INVALID_ARGUMENT": 5, + "CNDEV_ERROR_INVALID_DEVICE_ID": 6, + "CNDEV_ERROR_UNKNOWN": 7, + "CNDEV_ERROR_MALLOC": 8, + "CNDEV_ERROR_INSUFFICIENT_SPACE": 9, + "CNDEV_ERROR_NOT_SUPPORTED": 10, + "CNDEV_ERROR_INVALID_LINK": 11, + "CNDEV_ERROR_NO_DEVICES": 12, + "CNDEV_ERROR_NO_PERMISSION": 13, + "CNDEV_ERROR_NOT_FOUND": 14, + "CNDEV_ERROR_IN_USE": 15, + "CNDEV_ERROR_DUPLICATE": 16, + "CNDEV_ERROR_TIMEOUT": 17, + "CNDEV_ERROR_IN_PROBLEM": 18, +} + +# cndevNameEnum_t, limited to the types the card-name fallback table answers for: a wrong +# value there names the wrong card on a driver that exports neither name string. +HEADER_DEVICE_TYPES = { + "CNDEV_DEVICE_TYPE_MLU100": 0, + "CNDEV_DEVICE_TYPE_MLU270": 1, + "CNDEV_DEVICE_TYPE_MLU220_M2": 16, + "CNDEV_DEVICE_TYPE_MLU220_EDGE": 17, + "CNDEV_DEVICE_TYPE_MLU220_EVB": 18, + "CNDEV_DEVICE_TYPE_MLU220_M2i": 19, + "CNDEV_DEVICE_TYPE_MLU290": 20, + "CNDEV_DEVICE_TYPE_MLU370": 23, + "CNDEV_DEVICE_TYPE_MLU365": 24, + "CNDEV_DEVICE_TYPE_CE3226": 25, + "CNDEV_DEVICE_TYPE_MLU590": 26, + "CNDEV_DEVICE_TYPE_MLU585": 27, + "CNDEV_DEVICE_TYPE_MLU580": 30, + "CNDEV_DEVICE_TYPE_MLU570": 31, +} + +# What the fake library reports, so an assertion names a value rather than a literal. +FAKE_DEVICE_COUNT = 4 +FAKE_HANDLE_BASE = 100 +FAKE_UUID = "d4e5f60718293a4b" +FAKE_CARD_NAME = "MLU590-M9" +FAKE_MEMORY_TOTAL = 48000 +FAKE_MEMORY_USED = 1024 +FAKE_DRIVER_VERSION = (6, 10, 3) +FAKE_CORE_UTILIZATION = 37 +FAKE_BOARD_TEMPERATURE = 45 +FAKE_POWER_USAGE = 110 +FAKE_POWER_CAP = 250 +FAKE_NUMA_NODE = 1 +FAKE_CORRECTED_ERRORS = 2 +FAKE_PCIE_BUS_ID = "0000:3b:00.0" + +# A device handle the fake library answers for. +DEVICE = FAKE_HANDLE_BASE + + +def _identity(obj): + """ + Stand in for ctypes.byref. + + The fake library's entry points are plain Python callables, so a real byref() would + hand them a CArgObject that cannot be read or written from Python. Passing the + struct itself keeps what the wrapper sent -- and what it expects back -- visible. + """ + return obj + + +def _default_handlers(): + """ + Build one handler per libcndev.so entry point the binding may call. + + Each handler fills the out-struct the way a healthy MLU590 card would and returns + CNDEV_SUCCESS. + """ + + def init(reserved): + assert reserved is not None + return pycndev.CNDEV_SUCCESS + + def release(): + return pycndev.CNDEV_SUCCESS + + def get_device_count(card_info): + card_info.number = FAKE_DEVICE_COUNT + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_index(index, handle): + handle.value = FAKE_HANDLE_BASE + index + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_uuid(uuid, handle): + assert isinstance(uuid, bytes) + handle.value = FAKE_HANDLE_BASE + return pycndev.CNDEV_SUCCESS + + def get_device_handle_by_pci_bus_id(pci_bus_id, handle): + assert isinstance(pci_bus_id, bytes) + handle.value = FAKE_HANDLE_BASE + return pycndev.CNDEV_SUCCESS + + def get_uuid(uuid_info, device): + assert device == DEVICE + uuid_info.uuid = FAKE_UUID + return pycndev.CNDEV_SUCCESS + + def get_pcie_info_v2(pcie_info, device): + assert device == DEVICE + pcie_info.domain = 0x0000 + pcie_info.bus = 0x3B + pcie_info.device = 0x00 + pcie_info.function = 0 + return pycndev.CNDEV_SUCCESS + + def get_card_name(card_name, device): + assert device == DEVICE + card_name.id = pycndev.CNDEV_DEVICE_TYPE_MLU590 + return pycndev.CNDEV_SUCCESS + + def get_card_name_string(card_name_id): + assert card_name_id == pycndev.CNDEV_DEVICE_TYPE_MLU590 + return b"MLU590" + + def get_card_name_string_by_dev_id(device): + assert device == DEVICE + return FAKE_CARD_NAME.encode() + + def get_memory_usage_v2(memory_info, device): + assert device == DEVICE + memory_info.physicalMemoryTotal = FAKE_MEMORY_TOTAL + memory_info.physicalMemoryUsed = FAKE_MEMORY_USED + return pycndev.CNDEV_SUCCESS + + def get_card_health_state(health_state, device): + assert device == DEVICE + health_state.health = 1 + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + return pycndev.CNDEV_SUCCESS + + def get_card_health_state_v2(health_state, device): + assert device == DEVICE + health_state.health = 1 + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + health_state.overallHealth = pycndev.CNDEV_HEALTH_RESULT_PASS + return pycndev.CNDEV_SUCCESS + + def get_version_info(version_info, device): + assert device == DEVICE + major, minor, build = FAKE_DRIVER_VERSION + version_info.driverMajorVersion = major + version_info.driverMinorVersion = minor + version_info.driverBuildVersion = build + return pycndev.CNDEV_SUCCESS + + def get_device_utilization_info(util_info, device): + assert device == DEVICE + util_info.averageCoreUtilization = FAKE_CORE_UTILIZATION + return pycndev.CNDEV_SUCCESS + + def get_temperature_info(temp_info, device): + assert device == DEVICE + temp_info.board = FAKE_BOARD_TEMPERATURE + temp_info.chip = FAKE_BOARD_TEMPERATURE + return pycndev.CNDEV_SUCCESS + + def get_device_power_info(power_info, device): + assert device == DEVICE + power_info.usage = FAKE_POWER_USAGE + power_info.cap = FAKE_POWER_CAP + return pycndev.CNDEV_SUCCESS + + def get_numa_node_id(numa_node_id, device): + assert device == DEVICE + numa_node_id.nodeId = FAKE_NUMA_NODE + return pycndev.CNDEV_SUCCESS + + def get_ecc_info(ecc_info, device): + assert device == DEVICE + ecc_info.correctedError = FAKE_CORRECTED_ERRORS + return pycndev.CNDEV_SUCCESS + + return { + "cndevInit": init, + "cndevRelease": release, + "cndevGetDeviceCount": get_device_count, + "cndevGetDeviceHandleByIndex": get_device_handle_by_index, + "cndevGetDeviceHandleByUUID": get_device_handle_by_uuid, + "cndevGetDeviceHandleByPciBusId": get_device_handle_by_pci_bus_id, + "cndevGetUUID": get_uuid, + "cndevGetPCIeInfoV2": get_pcie_info_v2, + "cndevGetCardName": get_card_name, + "cndevGetCardNameString": get_card_name_string, + "cndevGetCardNameStringByDevId": get_card_name_string_by_dev_id, + "cndevGetMemoryUsageV2": get_memory_usage_v2, + "cndevGetCardHealthState": get_card_health_state, + "cndevGetCardHealthStateV2": get_card_health_state_v2, + "cndevGetVersionInfo": get_version_info, + "cndevGetDeviceUtilizationInfo": get_device_utilization_info, + "cndevGetTemperatureInfo": get_temperature_info, + "cndevGetDevicePowerInfo": get_device_power_info, + "cndevGetNUMANodeIdByDevId": get_numa_node_id, + "cndevGetECCInfo": get_ecc_info, + } + + +class FakeLibrary: + """ + A stand-in for libcndev.so that records what the binding asked of it. + + A symbol outside its handler set raises AttributeError, which is how a real CDLL + reports a driver too old to export a call. + """ + + def __init__(self, handlers: dict): + self.handlers = handlers + self.symbols: list[str] = [] + self.calls: list[tuple] = [] + + def __getattr__(self, name: str): + handler = self.handlers.get(name) + if handler is None: + raise AttributeError(name) + + self.symbols.append(name) + + def entry_point(*args): + self.calls.append((name, args)) + return handler(*args) + + return entry_point + + +def install_fake_library(monkeypatch, missing: tuple[str, ...] = ()) -> FakeLibrary: + """ + Put a FakeLibrary in place of the loaded libcndev.so, as if cndevInit() had run. + + Args: + monkeypatch: + The pytest monkeypatch fixture, which restores every global afterwards. + missing: + Symbols to withhold, standing in for a driver that does not export them. + + Returns: + The installed FakeLibrary. + + """ + handlers = { + name: handler + for name, handler in _default_handlers().items() + if name not in missing + } + lib = FakeLibrary(handlers) + + monkeypatch.setattr(pycndev, "cndevLib", lib) + monkeypatch.setattr(pycndev, "_libInitialized", True) + monkeypatch.setattr(pycndev, "_libInitializedException", None) + monkeypatch.setattr(pycndev, "_cndevGetFunctionPointer_cache", {}) + monkeypatch.setattr(pycndev, "byref", _identity) + + return lib + + +@pytest.mark.parametrize("name", list(EXPECTED_LAYOUTS)) +def test_struct_layout_matches_header(name): + expected_size, expected_fields = EXPECTED_LAYOUTS[name] + struct = getattr(pycndev, name) + + assert sizeof(struct) == expected_size + + # Field names as well as offsets: an extra field tucked into trailing padding leaves + # every offset and the total size untouched. + assert [field[0] for field in struct._fields_] == [ + field[0] for field in expected_fields + ] + + for field_name, offset, size in expected_fields: + descriptor = getattr(struct, field_name) + assert descriptor.offset == offset, field_name + assert descriptor.size == size, field_name + + +def test_api_version_matches_the_header(): + assert pycndev.CNDEV_VERSION_6 == HEADER_API_VERSION + + +@pytest.mark.parametrize("name, value", list(HEADER_ERROR_CODES.items())) +def test_error_code_matches_the_header(name, value): + assert getattr(pycndev, name) == value + + +@pytest.mark.parametrize("name, value", list(HEADER_DEVICE_TYPES.items())) +def test_device_type_matches_the_header(name, value): + assert getattr(pycndev, name) == value + + +def test_import_makes_no_library_call(monkeypatch): + """ + Importing the module must not reach for libcndev.so. + + The detector package is imported on every host, and all but a few have no Cambricon + driver at all. + """ + # Execute a second, privately named copy: the module registers its error subclasses + # on sys.modules[__name__], so re-executing it under the real name would rebind the + # classes the rest of this file raises and compares. + probe_name = f"{MODULE_NAME}_import_probe" + spec = importlib.util.spec_from_file_location(probe_name, pycndev.__file__) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, probe_name, module) + + with mock.patch( + "ctypes.CDLL", + side_effect=AssertionError("the library was loaded at import time"), + ): + spec.loader.exec_module(module) + + assert module.cndevLib is None + + +def test_missing_library_raises_cndev_error(monkeypatch): + """ + A host without libcndev.so must fail as CNDevError, not as a bare OSError. + + The detector catches the binding's own error type; an OSError escaping from here + would take the whole detection down instead of skipping the vendor. + """ + monkeypatch.setattr(pycndev, "cndevLib", None) + monkeypatch.setattr(pycndev, "_libInitialized", False) + monkeypatch.setattr(pycndev, "_libInitializedException", None) + monkeypatch.setattr( + pycndev, + "CDLL", + mock.Mock(side_effect=OSError("cannot open shared object file")), + ) + + with pytest.raises(pycndev.CNDevError) as excinfo: + pycndev.cndevInit() + + assert excinfo.value == pycndev.CNDEV_ERROR_LIBRARY_NOT_FOUND + assert not isinstance(excinfo.value, OSError) + assert str(excinfo.value) == "Library Not Found" + + +def test_uninitialized_call_raises_cndev_error(monkeypatch): + monkeypatch.setattr(pycndev, "cndevLib", None) + monkeypatch.setattr(pycndev, "_cndevGetFunctionPointer_cache", {}) + + with pytest.raises(pycndev.CNDevError) as excinfo: + pycndev.cndevGetDeviceCount() + + assert excinfo.value == pycndev.CNDEV_ERROR_UNINITIALIZED + + +def test_error_maps_code_to_subclass(): + err = pycndev.CNDevError(pycndev.CNDEV_ERROR_NOT_SUPPORTED) + + assert isinstance(err, pycndev.CNDevError_NotSupported) + assert err == pycndev.CNDEV_ERROR_NOT_SUPPORTED + assert err == pycndev.CNDevError(pycndev.CNDEV_ERROR_NOT_SUPPORTED) + assert err != pycndev.CNDevError(pycndev.CNDEV_ERROR_NO_DRIVER) + assert str(err) == "Not Supported" + + assert ( + pycndev.cndevExceptionClass(pycndev.CNDEV_ERROR_NO_DRIVER) + is pycndev.CNDevError_NoDriver + ) + + +def test_error_renders_an_unknown_code(): + assert str(pycndev.CNDevError(-12345)) == "Unknown CNDev Error -12345" + + +# One entry per wrapper that maps 1:1 onto a C entry point, so a wrapper reaching for +# the wrong symbol fails here rather than on a Cambricon host. +DIRECT_CALLS = [ + ("cndevGetDeviceCount", lambda: pycndev.cndevGetDeviceCount()), + ("cndevGetDeviceHandleByIndex", lambda: pycndev.cndevGetDeviceHandleByIndex(0)), + ("cndevGetDeviceHandleByUUID", lambda: pycndev.cndevGetDeviceHandleByUUID("MLU-x")), + ( + "cndevGetDeviceHandleByPciBusId", + lambda: pycndev.cndevGetDeviceHandleByPciBusId(FAKE_PCIE_BUS_ID), + ), + ("cndevGetUUID", lambda: pycndev.cndevGetUUID(DEVICE)), + ("cndevGetPCIeInfoV2", lambda: pycndev.cndevGetPCIeInfoV2(DEVICE)), + ("cndevGetCardName", lambda: pycndev.cndevGetCardName(DEVICE)), + ( + "cndevGetCardNameString", + lambda: pycndev.cndevGetCardNameString(pycndev.CNDEV_DEVICE_TYPE_MLU590), + ), + ( + "cndevGetCardNameStringByDevId", + lambda: pycndev.cndevGetCardNameStringByDevId(DEVICE), + ), + ("cndevGetMemoryUsageV2", lambda: pycndev.cndevGetMemoryUsageV2(DEVICE)), + ("cndevGetCardHealthState", lambda: pycndev.cndevGetCardHealthState(DEVICE)), + ("cndevGetCardHealthStateV2", lambda: pycndev.cndevGetCardHealthStateV2(DEVICE)), + ("cndevGetVersionInfo", lambda: pycndev.cndevGetVersionInfo(DEVICE)), + ( + "cndevGetDeviceUtilizationInfo", + lambda: pycndev.cndevGetDeviceUtilizationInfo(DEVICE), + ), + ("cndevGetTemperatureInfo", lambda: pycndev.cndevGetTemperatureInfo(DEVICE)), + ("cndevGetDevicePowerInfo", lambda: pycndev.cndevGetDevicePowerInfo(DEVICE)), + ("cndevGetNUMANodeIdByDevId", lambda: pycndev.cndevGetNUMANodeIdByDevId(DEVICE)), + ("cndevGetECCInfo", lambda: pycndev.cndevGetECCInfo(DEVICE)), +] + +# The wrappers whose out-struct carries the header's IN version field. cndev rejects a +# versioned struct that does not declare which layout the caller speaks, so an unstamped +# one is answered with CNDEV_ERROR_UNSUPPORTED_API_VERSION rather than data. +VERSIONED_CALLS = [ + entry + for entry in DIRECT_CALLS + if entry[0] + in { + "cndevGetDeviceCount", + "cndevGetUUID", + "cndevGetCardName", + "cndevGetCardHealthState", + "cndevGetVersionInfo", + "cndevGetDeviceUtilizationInfo", + "cndevGetTemperatureInfo", + "cndevGetNUMANodeIdByDevId", + "cndevGetECCInfo", + } +] + + +@pytest.mark.parametrize("symbol, call", DIRECT_CALLS) +def test_wrapper_calls_expected_symbol(monkeypatch, symbol, call): + lib = install_fake_library(monkeypatch) + + call() + + assert [name for name, _ in lib.calls] == [symbol] + + +@pytest.mark.parametrize("symbol, call", VERSIONED_CALLS) +def test_versioned_call_declares_the_api_version(monkeypatch, symbol, call): + lib = install_fake_library(monkeypatch) + + call() + + _, args = lib.calls[-1] + assert args[0].version == HEADER_API_VERSION, symbol + + +@pytest.mark.parametrize("symbol, call", DIRECT_CALLS) +def test_wrapper_raises_on_driver_failure(monkeypatch, symbol, call): + if symbol in ("cndevGetCardNameString", "cndevGetCardNameStringByDevId"): + pytest.skip("returns a string, not a cndevRet_t") + + lib = install_fake_library(monkeypatch) + lib.handlers[symbol] = lambda *_: pycndev.CNDEV_ERROR_NOT_SUPPORTED + + with pytest.raises(pycndev.CNDevError_NotSupported): + call() + + +def test_get_device_count_returns_the_reported_number(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetDeviceCount() == FAKE_DEVICE_COUNT + + +def test_get_device_handle_by_index_returns_the_handle(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetDeviceHandleByIndex(2) == FAKE_HANDLE_BASE + 2 + + +def test_get_uuid_prefixes_the_manufacturer_tag(monkeypatch): + """ + The UUID must read the way every Cambricon tool and the operator report it. + """ + install_fake_library(monkeypatch) + + assert pycndev.cndevGetUUID(DEVICE) == "MLU-" + FAKE_UUID + + +def test_get_uuid_rejects_an_empty_field(monkeypatch): + """ + A driver answering success with an empty UUID has reported no identity. + + Prefixing it would yield a bare "MLU-" that reads as a valid id and is the same + for every card, so the usage join would write one card's metrics onto all of them. + """ + lib = install_fake_library(monkeypatch) + + def _empty_uuid(uuid_info, _device): + uuid_info.uuid = "" + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetUUID"] = _empty_uuid + + with pytest.raises(pycndev.CNDevError) as raised: + pycndev.cndevGetUUID(DEVICE) + + assert raised.value == pycndev.CNDEV_ERROR_NOT_FOUND + + +def test_get_pcie_bus_id_formats_the_bdf(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetPCIeBusId(DEVICE) == FAKE_PCIE_BUS_ID + + +def test_get_card_name_prefers_the_driver_string(monkeypatch): + lib = install_fake_library(monkeypatch) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == FAKE_CARD_NAME + # The cheapest call answered, so the enum is never consulted. + assert "cndevGetCardName" not in lib.symbols + + +def test_get_card_name_falls_back_to_the_name_enum(monkeypatch): + lib = install_fake_library(monkeypatch, missing=("cndevGetCardNameStringByDevId",)) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU590" + assert "cndevGetCardNameString" in lib.symbols + + +def test_get_card_name_falls_back_to_the_name_table(monkeypatch): + """ + A driver exporting neither name string still yields a card name. + """ + lib = install_fake_library( + monkeypatch, + missing=("cndevGetCardNameStringByDevId", "cndevGetCardNameString"), + ) + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU590" + assert "cndevGetCardName" in lib.symbols + + +def test_get_card_name_falls_back_to_the_family(monkeypatch): + lib = install_fake_library( + monkeypatch, + missing=("cndevGetCardNameStringByDevId", "cndevGetCardNameString"), + ) + + def unknown_card_name(card_name, _device): + card_name.id = 0x7FFF + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetCardName"] = unknown_card_name + + assert pycndev.cndevGetCardNameByDevId(DEVICE) == "MLU" + + +def test_get_memory_usage_reports_the_physical_totals(monkeypatch): + install_fake_library(monkeypatch) + + memory_info = pycndev.cndevGetMemoryUsageV2(DEVICE) + + assert memory_info.physicalMemoryTotal == FAKE_MEMORY_TOTAL + assert memory_info.physicalMemoryUsed == FAKE_MEMORY_USED + + +def test_get_card_health_state_v2_falls_back_to_v1(monkeypatch): + """ + A driver without the V2 call still reports health, through the V1 struct. + """ + lib = install_fake_library(monkeypatch, missing=("cndevGetCardHealthStateV2",)) + + health_state = pycndev.cndevGetCardHealthStateV2(DEVICE) + + assert "cndevGetCardHealthState" in lib.symbols + assert health_state.health == 1 + assert health_state.deviceState == pycndev.CNDEV_HEALTH_STATE_DEVICE_GOOD + assert health_state.driverState == pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + # V1 knows nothing about incidents, so the report stays empty rather than stale. + assert health_state.incident_count == 0 + # And the overall verdict is derived rather than left at zero, which is + # CNDEV_HEALTH_RESULT_PASS and would read as a pass on every old driver. + assert health_state.overallHealth == pycndev.CNDEV_HEALTH_RESULT_PASS + + +def test_get_card_health_state_v2_derives_a_failing_v1_verdict(monkeypatch): + lib = install_fake_library(monkeypatch, missing=("cndevGetCardHealthStateV2",)) + + def _in_problem(health_state, _device): + health_state.health = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + health_state.deviceState = pycndev.CNDEV_HEALTH_STATE_DEVICE_IN_PROBLEM + health_state.driverState = pycndev.CNDEV_HEALTH_STATE_DRIVER_RUNNING + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevGetCardHealthState"] = _in_problem + + health_state = pycndev.cndevGetCardHealthStateV2(DEVICE) + + assert health_state.overallHealth == pycndev.CNDEV_HEALTH_RESULT_FAIL + + +def test_get_version_info_reports_the_driver_version(monkeypatch): + install_fake_library(monkeypatch) + + version_info = pycndev.cndevGetVersionInfo(DEVICE) + + assert ( + version_info.driverMajorVersion, + version_info.driverMinorVersion, + version_info.driverBuildVersion, + ) == FAKE_DRIVER_VERSION + + +def test_usage_wrappers_report_their_metrics(monkeypatch): + install_fake_library(monkeypatch) + + assert ( + pycndev.cndevGetDeviceUtilizationInfo(DEVICE).averageCoreUtilization + == FAKE_CORE_UTILIZATION + ) + assert pycndev.cndevGetTemperatureInfo(DEVICE).board == FAKE_BOARD_TEMPERATURE + assert pycndev.cndevGetDevicePowerInfo(DEVICE).usage == FAKE_POWER_USAGE + assert pycndev.cndevGetDevicePowerInfo(DEVICE).cap == FAKE_POWER_CAP + + +def test_get_numa_node_id_reports_the_node(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetNUMANodeIdByDevId(DEVICE).nodeId == FAKE_NUMA_NODE + + +def test_get_ecc_info_reports_the_counters(monkeypatch): + install_fake_library(monkeypatch) + + assert pycndev.cndevGetECCInfo(DEVICE).correctedError == FAKE_CORRECTED_ERRORS + + +def test_init_is_idempotent(monkeypatch): + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + pycndev.cndevInit() + pycndev.cndevInit() + + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_init_replays_a_cached_failure(monkeypatch): + """ + A failed init must keep failing, with a fresh exception each time. + + Re-raising the cached object appends a traceback frame per raise, and those frames + retain their callers' locals. See gpustack/gpustack#5342. + """ + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + lib.handlers["cndevInit"] = lambda *_: pycndev.CNDEV_ERROR_NO_DRIVER + + with pytest.raises(pycndev.CNDevError) as first: + pycndev.cndevInit() + with pytest.raises(pycndev.CNDevError) as second: + pycndev.cndevInit() + + assert first.value == pycndev.CNDEV_ERROR_NO_DRIVER + assert second.value == pycndev.CNDEV_ERROR_NO_DRIVER + assert first.value is not second.value + # The driver was asked once; the second call replayed the cached failure. + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_init_reaches_the_driver_once_when_two_callers_race(monkeypatch): + """ + Two concurrent first callers must not both reach the driver. + + The loser is answered with an already-initialized error, and caching that as the + library's permanent state disables Cambricon detection until the process restarts. + The operator's binding uses sync.Once here. + """ + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + entered = threading.Event() + entered_again = threading.Event() + proceed = threading.Event() + entries: list[int] = [] + + def _blocking_init(*_args): + entries.append(1) + (entered if len(entries) == 1 else entered_again).set() + # Stay inside the driver call until the second caller has had its chance. + proceed.wait(timeout=5) + return pycndev.CNDEV_SUCCESS + + lib.handlers["cndevInit"] = _blocking_init + + failures: list[BaseException] = [] + + def _init(): + try: + pycndev.cndevInit() + except BaseException as e: + failures.append(e) + + first = threading.Thread(target=_init) + first.start() + assert entered.wait(timeout=5) + + second = threading.Thread(target=_init) + second.start() + # The second caller has to wait for the lock the first one holds, so it must not + # reach the driver while the first call is still open. + assert not entered_again.wait(timeout=0.5) + + proceed.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not failures + assert [name for name, _ in lib.calls] == ["cndevInit"] + + +def test_release_reaches_the_driver(monkeypatch): + lib = install_fake_library(monkeypatch) + + pycndev.cndevRelease() + + assert [name for name, _ in lib.calls] == ["cndevRelease"] + + +def test_release_without_init_is_a_no_op(monkeypatch): + lib = install_fake_library(monkeypatch) + monkeypatch.setattr(pycndev, "_libInitialized", False) + + pycndev.cndevRelease() + + assert lib.calls == [] + + +def test_release_tolerates_a_driver_without_the_symbol(monkeypatch): + install_fake_library(monkeypatch, missing=("cndevRelease",)) + + pycndev.cndevRelease() diff --git a/tests/gpustack_runtime/detector/test_samples.py b/tests/gpustack_runtime/detector/test_samples.py new file mode 100644 index 0000000..d37f681 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_samples.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from typing import Any + +import pytest + +from gpustack_runtime.detector import Device +from gpustack_runtime.detector.__types__ import ( + DeviceMemoryStatusEnum, + ManufacturerEnum, +) + +_SAMPLES_PATH = Path(__file__).parent / "samples" + +_DEVICE_FIELDS = {field.name for field in dataclasses.fields(Device)} + +_RETIRED_APPENDIX_KEYS = ( + # Whole-card reporting: no virtual/PF/VF classification. + "vgpu", + # MIG instances are appendix entries of their card, not devices of their + # own, so they no longer carry a device index per instance handle. + "gpu_instance_index", + "compute_instance_index", +) +""" +Appendix keys no detector emits anymore, which a sample must not resurrect. +""" + + +def _samples() -> list[Path]: + """ + The detect output samples. `topology_output_*.json` describes a Topology, + not a Device, and is out of this guard's scope. + """ + return sorted(_SAMPLES_PATH.glob("detect_output_*.json")) + + +def _load(sample: Path) -> list[dict[str, Any]]: + devices = json.loads(sample.read_text()) + assert devices, f"{sample.name} carries no device" + return devices + + +def _mig_devices(device: dict[str, Any]) -> list[dict[str, Any]]: + return (device.get("appendix") or {}).get("mig_devices") or [] + + +def _assert_deserializes(sample: Path, entry: dict[str, Any]) -> None: + unknown = sorted(set(entry) - _DEVICE_FIELDS) + assert not unknown, f"{sample.name} carries fields Device cannot hold: {unknown}" + + device = Device.from_dict(entry) + assert isinstance(device.manufacturer, ManufacturerEnum) + assert isinstance(device.memory_status, DeviceMemoryStatusEnum) + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_deserializes_into_device(sample: Path): + for entry in _load(sample): + _assert_deserializes(sample, entry) + for mig_device in _mig_devices(entry): + _assert_deserializes(sample, mig_device) + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_carries_no_retired_appendix_key(sample: Path): + for entry in _load(sample): + appendices = [entry.get("appendix") or {}] + appendices += [ + mig_device.get("appendix") or {} for mig_device in _mig_devices(entry) + ] + for appendix in appendices: + retired = sorted(set(appendix) & set(_RETIRED_APPENDIX_KEYS)) + assert not retired, f"{sample.name} carries retired keys: {retired}" + + +@pytest.mark.parametrize("sample", _samples(), ids=lambda sample: sample.name) +def test_sample_keeps_mig_instances_inside_the_card(sample: Path): + devices = _load(sample) + indexes = [entry["index"] for entry in devices] + + for entry in devices: + appendix = entry.get("appendix") or {} + assert not appendix.get("sliced"), ( + f"{sample.name} reports a MIG instance as a device of its own" + ) + if appendix.get("mig"): + assert "mig_devices" in appendix, ( + f"{sample.name} enables MIG without reporting its instances" + ) + for mig_device in _mig_devices(entry): + assert mig_device["appendix"]["sliced"] + # index_mig_devices numbers the instances above every card. + assert mig_device["index"] > max(indexes) diff --git a/tests/gpustack_runtime/detector/test_thead.py b/tests/gpustack_runtime/detector/test_thead.py new file mode 100644 index 0000000..0a27761 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_thead.py @@ -0,0 +1,830 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from types import SimpleNamespace + +import pytest + +from gpustack_runtime.deployer.cdi import thead as cdi_thead +from gpustack_runtime.deployer.cdi.thead import THeadGenerator +from gpustack_runtime.detector import pyhgml, thead +from gpustack_runtime.detector.__types__ import ( + Device, + DeviceMemoryStatusEnum, + ManufacturerEnum, +) +from gpustack_runtime.detector.thead import THeadDetector + + +@pytest.mark.skipif( + not THeadDetector.is_supported(), + reason="T-Head PPU not detected", +) +def test_detect(): + det = THeadDetector() + devs = det.detect() + print(devs) + + +@pytest.mark.skipif( + not THeadDetector.is_supported(), + reason="T-Head PPU not detected", +) +def test_get_topology(): + det = THeadDetector() + topo = det.get_topology() + print(topo) + + +# --------------------------------------------------------------------------- # +# A fake pyhgml, so the split is provable on a host with no T-Head driver. # +# --------------------------------------------------------------------------- # + +_MIB = 1 << 20 + + +@dataclass +class _Instance: + """ + One GPU/compute instance of a MIG-enabled card, as the driver reports it. + + The runtime enumerates these where the operator does not: the physical card + stays the reported device and its instances live in + ``appendix["mig_devices"]``. + """ + + uuid: str + gpu_instance_id: int = 0 + compute_instance_id: int = 0 + memory_total: int = 8192 * _MIB + memory_used: int = 512 * _MIB + ecc_errors: int = 0 + sm_util: float | None = 60.0 + """ + The SM utilization GPM samples for the instance, or None when unreadable. + """ + uuid_readable: bool = True + """ + Whether the driver answers this instance's UUID, i.e. whether it is the one + faulty instance of an otherwise healthy card. + """ + + +@dataclass +class _Card: + """ + One card as the driver would report it, in the driver's own units. + """ + + uuid: str + name: str = "T-Head PPU" + bdf: str = "0000:01:00.0" + minor_number: int | None = 3 + compute_capability: tuple[int, int] = (8, 0) + cores: int = 128 + memory_total: int = 65536 * _MIB + memory_used: int = 1024 * _MIB + ecc_errors: int = 0 + cores_utilization: int = 33 + temperature: int = 55 + power_limit: int = 350_000 # mW + power_usage: int = 150_000 # mW + instances: list[_Instance] | None = None + """ + The card's GPU/compute instances, or None for a card with MIG disabled. + """ + + +class _GpmMetrics: + """ + A GPM metrics request, as the detector fills and reads it. + """ + + def __init__(self): + self.version = 0 + self.numMetrics = 0 + self.sample1 = None + self.sample2 = None + self.metrics = [SimpleNamespace(metricId=0, value=float("nan"))] + + +@dataclass +class _FakeHGML: + """ + A stand-in for the pyhgml binding, recording every call the detector makes. + + The error type is the real module's and every HGML_* constant is delegated + to it, so a fake drifting from the binding's contract fails here rather than + on hardware. Entry points are dispatched by name, as the fake pymxsml of + test_metax.py does, which keeps HGML's camelCase out of the handlers' own + names -- pyhgml's surface is wide enough for that to matter. + """ + + cards: list[_Card] + calls: list[str] = field(default_factory=list) + gpm_supported: bool = True + + HGMLError = pyhgml.HGMLError + + c_hgmlGpmMetricsGet_t = _GpmMetrics # noqa: N815 + + instance_profile_name = "1g.8gb" + instance_profile_memory_mb = 8192 + instance_profile_cores = 16 + + def __getattr__(self, name: str): + handler = { + "hgmlInit": self._init, + "hgmlSystemGetDriverVersion": self._system_get_driver_version, + "hgmlSystemGetHggcDriverVersion": self._system_get_hggc_driver_version, + "hgmlDeviceGetCount": self._device_get_count, + "hgmlDeviceGetHandleByIndex": self._device_get_handle_by_index, + "hgmlDeviceGetHggcComputeCapability": self._device_get_compute_capability, + "hgmlDeviceGetPciInfo": self._device_get_pci_info, + "hgmlDeviceGetMemoryAffinity": self._device_get_memory_affinity, + "hgmlDeviceGetMinorNumber": self._device_get_minor_number, + "hgmlDeviceGetName": self._device_get_name, + "hgmlDeviceGetUUID": self._device_get_uuid, + "hgmlDeviceGetNumGpuCores": self._device_get_num_gpu_cores, + "hgmlDeviceGetMemoryInfo": self._device_get_memory_info, + "hgmlDeviceGetMemoryErrorCounter": self._device_get_memory_error_counter, + "hgmlDeviceGetPowerManagementDefaultLimit": self._device_get_power_limit, + "hgmlDeviceGetUtilizationRates": self._device_get_utilization_rates, + "hgmlDeviceGetTemperature": self._device_get_temperature, + "hgmlDeviceGetPowerUsage": self._device_get_power_usage, + "hgmlDeviceGetMigMode": self._device_get_mig_mode, + "hgmlDeviceGetMaxMigDeviceCount": self._device_get_max_mig_device_count, + "hgmlDeviceGetMigDeviceHandleByIndex": self._device_get_mig_handle, + "hgmlDeviceGetGpuInstanceId": self._device_get_gpu_instance_id, + "hgmlDeviceGetComputeInstanceId": self._device_get_compute_instance_id, + "hgmlDeviceGetGpuInstanceById": self._device_get_gpu_instance_by_id, + "hgmlGpuInstanceGetComputeInstanceById": self._gi_get_ci_by_id, + "hgmlGpuInstanceGetInfo": self._gi_get_info, + "hgmlComputeInstanceGetInfo": self._ci_get_info, + "hgmlDeviceGetGpuInstanceProfileInfo": self._device_get_gi_profile_info, + "hgmlGpuInstanceGetComputeInstanceProfileInfo": self._gi_get_ci_profile_info, + "hgmlGpmQueryDeviceSupport": self._gpm_query_device_support, + "hgmlGpmSampleAlloc": self._gpm_sample_alloc, + "hgmlGpmSampleFree": self._gpm_sample_free, + "hgmlGpmSampleGet": self._gpm_sample_get, + "hgmlGpmMigSampleGet": self._gpm_mig_sample_get, + "hgmlGpmMetricsGet": self._gpm_metrics_get, + }.get(name) + if handler is None: + if name.startswith("HGML_"): + # Enumeration constants are the real module's, so a fake + # drifting from the binding's contract fails here rather than + # on hardware. + return getattr(pyhgml, name) + msg = f"module pyhgml has no attribute {name}" + raise AttributeError(msg) + + def entry_point(*args, **kwargs): + self.calls.append(name) + return handler(*args, **kwargs) + + return entry_point + + # System. + + def _init(self) -> None: + pass + + def _system_get_driver_version(self) -> str: + return "1.2.3" + + def _system_get_hggc_driver_version(self) -> int: + return 12030 # i.e. 12.3.0 + + def _device_get_count(self) -> int: + return len(self.cards) + + def _device_get_handle_by_index(self, index: int) -> _Card: + return self.cards[index] + + # Identity and capability. + + def _device_get_compute_capability(self, handle: _Card) -> tuple[int, int]: + return handle.compute_capability + + def _device_get_pci_info(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(busIdLegacy=handle.bdf) + + def _device_get_memory_affinity( + self, + handle: _Card, + node_set_size: int, + scope: int, + ) -> list[int]: + msg = "no memory affinity" + raise pyhgml.HGMLError(msg) + + def _device_get_minor_number(self, handle: _Card) -> int: + if handle.minor_number is None: + msg = "no minor number" + raise pyhgml.HGMLError(msg) + return handle.minor_number + + def _device_get_name(self, handle: _Card) -> str: + return handle.name + + def _device_get_uuid(self, handle: _Card | _Instance) -> str: + if not getattr(handle, "uuid_readable", True): + raise self.HGMLError(pyhgml.HGML_ERROR_NOT_FOUND) + return handle.uuid + + def _device_get_num_gpu_cores(self, handle: _Card) -> int: + return handle.cores + + def _device_get_power_limit(self, handle: _Card) -> int: + return handle.power_limit + + # Memory. + + def _device_get_memory_info(self, handle: _Card | _Instance) -> SimpleNamespace: + return SimpleNamespace( + total=handle.memory_total, + used=handle.memory_used, + ) + + def _device_get_memory_error_counter( + self, + handle: _Card | _Instance, + error_type: int, + counter_type: int, + location_type: int, + ) -> int: + return handle.ecc_errors + + # Usage. + + def _device_get_utilization_rates(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(gpu=handle.cores_utilization, memory=0) + + def _device_get_temperature(self, handle: _Card, sensor: int) -> int: + return handle.temperature + + def _device_get_power_usage(self, handle: _Card) -> int: + return handle.power_usage + + def _gpm_query_device_support(self, handle: _Card) -> SimpleNamespace: + return SimpleNamespace(isSupportedDevice=int(self.gpm_supported)) + + def _gpm_sample_alloc(self) -> object: + return object() + + def _gpm_sample_free(self, sample: object) -> None: + pass + + def _gpm_sample_get(self, handle: _Card, sample: object) -> None: + self._gpm_target = handle + + def _gpm_mig_sample_get( + self, + handle: _Card, + gpu_instance_id: int, + sample: object, + ) -> None: + self._gpm_target = next( + ( + inst + for inst in handle.instances or [] + if inst.gpu_instance_id == gpu_instance_id + ), + None, + ) + + def _gpm_metrics_get(self, metrics_get: _GpmMetrics) -> None: + sm_util = getattr(self._gpm_target, "sm_util", None) + metrics_get.metrics[0].value = ( + float("nan") if sm_util is None else float(sm_util) + ) + + # GPU/compute instances. + + def _device_get_mig_mode(self, handle: _Card) -> tuple[int, int]: + mode = ( + pyhgml.HGML_DEVICE_MIG_DISABLE + if handle.instances is None + else pyhgml.HGML_DEVICE_MIG_ENABLE + ) + return mode, mode + + def _device_get_max_mig_device_count(self, handle: _Card) -> int: + return 8 + + def _device_get_mig_handle(self, handle: _Card, index: int) -> _Instance: + instances = handle.instances or [] + if index >= len(instances): + msg = "no instance at that slot" + raise pyhgml.HGMLError(msg) + return instances[index] + + def _device_get_gpu_instance_id(self, handle: _Instance) -> int: + return handle.gpu_instance_id + + def _device_get_compute_instance_id(self, handle: _Instance) -> int: + return handle.compute_instance_id + + def _device_get_gpu_instance_by_id( + self, + handle: _Card, + gpu_instance_id: int, + ) -> SimpleNamespace: + return SimpleNamespace(card=handle, gpu_instance_id=gpu_instance_id) + + def _gi_get_ci_by_id( + self, + gpu_instance: SimpleNamespace, + compute_instance_id: int, + ) -> SimpleNamespace: + return SimpleNamespace(compute_instance_id=compute_instance_id) + + def _gi_get_info(self, gpu_instance: SimpleNamespace) -> SimpleNamespace: + return SimpleNamespace(profileId=0) + + def _ci_get_info(self, compute_instance: SimpleNamespace) -> SimpleNamespace: + return SimpleNamespace(profileId=0) + + def _device_get_gi_profile_info( + self, + handle: _Card, + profile_id: int, + ) -> SimpleNamespace: + # Only the 1-slice profile exists, i.e. HGML_GPU_INSTANCE_PROFILE_1_SLICE. + if profile_id != pyhgml.HGML_GPU_INSTANCE_PROFILE_1_SLICE: + msg = "no such GPU instance profile" + raise pyhgml.HGMLError(msg) + return SimpleNamespace( + id=0, + memorySizeMB=self.instance_profile_memory_mb, + name=self.instance_profile_name, + ) + + def _gi_get_ci_profile_info( + self, + gpu_instance: SimpleNamespace, + profile_id: int, + engine_profile_id: int, + ) -> SimpleNamespace: + if (profile_id, engine_profile_id) != ( + pyhgml.HGML_COMPUTE_INSTANCE_PROFILE_1_SLICE, + pyhgml.HGML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED, + ): + msg = "no such compute instance profile" + raise pyhgml.HGMLError(msg) + return SimpleNamespace(id=0, multiprocessorCount=self.instance_profile_cores) + + +_USAGE_CALLS = ( + "hgmlGpmQueryDeviceSupport", + "hgmlGpmSampleGet", + "hgmlGpmMigSampleGet", + "hgmlGpmMetricsGet", + "hgmlDeviceGetUtilizationRates", + "hgmlDeviceGetTemperature", + "hgmlDeviceGetPowerUsage", +) +""" +The driver calls only the usage query is allowed to make. Deliberately not the +whole metric-looking surface: hgmlDeviceGetPowerManagementDefaultLimit reads the +power *limit* and hgmlDeviceGetMemoryInfo the memory *total*, both inventory. +""" + +_CARD_MEMORY_UTILIZATION = 1.56 +""" +A default card's memory utilization: 1024 MiB used of 65536 MiB total, as +get_utilization rounds it. +""" + +_INSTANCE_MEMORY_UTILIZATION = 6.25 +""" +A default instance's memory utilization: 512 MiB used of 8192 MiB total. +""" + + +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the ECC error check on: it is opt-in, as reading the counters costs a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults + to true. A real module attribute is set because the env lookup is cached. + """ + monkeypatch.setattr( + thead.envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + +@pytest.fixture +def detector(monkeypatch): + """ + Build a T-Head detector talking to a fake driver reporting the given cards. + """ + + def _install(*cards: _Card, **kwargs) -> tuple[THeadDetector, _FakeHGML]: + fake = _FakeHGML(cards=list(cards), **kwargs) + monkeypatch.setattr(thead, "pyhgml", fake) + # is_supported() initializes the real driver, which the fake replaces. + monkeypatch.setattr(THeadDetector, "is_supported", staticmethod(lambda: True)) + # The NUMA node comes from sysfs, so it is answered here instead of + # letting the host decide what the test sees. + monkeypatch.setattr(thead, "get_numa_node_by_bdf", lambda *_: "") + # GPM samples over a 100 ms window of real time, twice per query. + monkeypatch.setattr(thead.time, "sleep", lambda *_: None) + return THeadDetector(), fake + + return _install + + +# --------------------------------------------------------------------------- # +# detect_info: inventory only, no vgpu, no usage call. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_carries_the_inventory(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + dev = det.detect_info()[0] + + assert dev.manufacturer == ManufacturerEnum.THEAD + assert dev.index == 0 + assert dev.name == "T-Head PPU" + assert dev.uuid == "PPU-0" + assert dev.driver_version == "1.2.3" + assert dev.runtime_version == "12.3" + assert dev.runtime_version_original == "12.3.0" + assert dev.compute_capability == "8.0" + assert dev.cores == 128 + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert dev.appendix["bdf"] == "0000:01:00.0" + assert dev.appendix["mig"] is False + # The usage fields keep their defaults: the information query does not fill + # them, and must not invent a zero that reads like a measurement. + assert dev.cores_utilization == 0 + assert dev.memory_used == 0 + assert dev.memory_utilization == 0 + assert dev.temperature is None + assert dev.power_used is None + + +def test_detect_info_issues_no_usage_call(detector): + det, fake = detector( + _Card(uuid="PPU-0"), + _Card(uuid="PPU-1", instances=[_Instance(uuid="PPU-1-MIG-0")]), + ) + + det.detect_info() + + assert [call for call in fake.calls if call in _USAGE_CALLS] == [] + # The inventory reads that only look like metrics stay. + assert "hgmlDeviceGetPowerManagementDefaultLimit" in fake.calls + assert "hgmlDeviceGetMemoryInfo" in fake.calls + + +def test_no_appendix_carries_vgpu(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card(uuid="PPU-1", instances=[_Instance(uuid="PPU-1-MIG-0")]), + ) + + for dev in det.detect(): + assert "vgpu" not in dev.appendix + for inst in dev.appendix.get("mig_devices", []): + assert "vgpu" not in inst["appendix"] + + +# --------------------------------------------------------------------------- # +# The GPU/compute instance entries, which the operator has no equivalent of. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_keeps_the_instance_entries(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card( + uuid="PPU-1", + instances=[ + _Instance(uuid="PPU-1-MIG-0"), + _Instance(uuid="PPU-1-MIG-1", gpu_instance_id=1), + ], + ), + ) + + devices = det.detect_info() + + # The physical card stays the reported device, whether or not MIG is on. + assert [dev.uuid for dev in devices] == ["PPU-0", "PPU-1"] + assert "mig_devices" not in devices[0].appendix + + instances = devices[1].appendix["mig_devices"] + assert [inst["uuid"] for inst in instances] == ["PPU-1-MIG-0", "PPU-1-MIG-1"] + # `sliced` is what the topology path keys off, so it must survive. + assert all(inst["appendix"]["sliced"] is True for inst in instances) + assert all(inst["appendix"]["mig"] is True for inst in instances) + assert all(inst["appendix"]["bdf"] == "0000:01:00.0" for inst in instances) + assert [inst["appendix"]["gpu_instance_id"] for inst in instances] == [0, 1] + assert [inst["name"] for inst in instances] == ["1g.8gb"] * 2 + assert [inst["cores"] for inst in instances] == [16] * 2 + assert [inst["memory"] for inst in instances] == [8192] * 2 + # index_mig_devices numbers them above the cards, one block per card. + assert [inst["index"] for inst in instances] == [2 + 1 * 8, 2 + 1 * 8 + 1] + # The instances' usage fields keep a Device's defaults as well. + assert [inst["cores_utilization"] for inst in instances] == [0] * 2 + assert [inst["memory_used"] for inst in instances] == [0] * 2 + assert [inst["temperature"] for inst in instances] == [None] * 2 + assert [inst["power_used"] for inst in instances] == [None] * 2 + + +# --------------------------------------------------------------------------- # +# The minor number, recorded only when the driver answers. # +# --------------------------------------------------------------------------- # + + +def test_detect_info_records_the_minor_number(detector): + det, _ = detector(_Card(uuid="PPU-0", minor_number=3)) + + dev = det.detect_info()[0] + + assert dev.appendix["minor_number"] == 3 + # Device.index stays the enumeration index. + assert dev.index == 0 + + +def test_detect_info_omits_the_minor_number_when_unreadable(detector): + det, _ = detector(_Card(uuid="PPU-0", minor_number=None)) + + dev = det.detect_info()[0] + + # Absent rather than substituted by the enumeration index, as the operator + # does: a substituted value would make a wrong ordinal look proven. + assert "minor_number" not in dev.appendix + + +# --------------------------------------------------------------------------- # +# detect_usage: the six fields, merged by UUID, instances included. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_by_uuid(detector): + det, _ = detector( + _Card(uuid="PPU-0"), + _Card( + uuid="PPU-1", + cores_utilization=7, + memory_used=2048 * _MIB, + temperature=61, + power_usage=200_000, + ), + ) + + devices = det.detect_info() + # Reversed on purpose: the merge joins by UUID, never by position. + devices.reverse() + + assert det.detect_usage(devices) is devices + + by_uuid = {dev.uuid: dev for dev in devices} + assert by_uuid["PPU-0"].cores_utilization == 33 + assert by_uuid["PPU-0"].memory_used == 1024 + assert by_uuid["PPU-0"].memory_utilization == _CARD_MEMORY_UTILIZATION + assert by_uuid["PPU-0"].temperature == 55 + assert by_uuid["PPU-0"].power_used == 150 + assert by_uuid["PPU-1"].cores_utilization == 7 + assert by_uuid["PPU-1"].memory_used == 2048 + assert by_uuid["PPU-1"].temperature == 61 + assert by_uuid["PPU-1"].power_used == 200 + # The information fields survive the merge untouched. + assert by_uuid["PPU-0"].index == 0 + assert by_uuid["PPU-0"].memory == 65536 + assert by_uuid["PPU-0"].power == 350 + assert by_uuid["PPU-0"].name == "T-Head PPU" + + +def test_detect_usage_merges_the_instance_entries(detector): + det, _ = detector( + _Card( + uuid="PPU-0", + instances=[ + _Instance(uuid="PPU-0-MIG-0", sm_util=42.0), + _Instance(uuid="PPU-0-MIG-1", gpu_instance_id=1, sm_util=None), + ], + ), + ) + + devices = det.detect() + + instances = devices[0].appendix["mig_devices"] + assert [inst["cores_utilization"] for inst in instances] == [42, None] + assert [inst["memory_used"] for inst in instances] == [512] * 2 + assert [inst["memory_utilization"] for inst in instances] == [ + _INSTANCE_MEMORY_UTILIZATION, + ] * 2 + # An instance reports neither temperature nor power, so it carries the card's. + assert [inst["temperature"] for inst in instances] == [55] * 2 + assert [inst["power_used"] for inst in instances] == [150] * 2 + # The inventory fields survive the merge untouched. + assert [inst["name"] for inst in instances] == ["1g.8gb"] * 2 + assert [inst["memory"] for inst in instances] == [8192] * 2 + assert all(inst["appendix"]["sliced"] is True for inst in instances) + + +def test_detect_usage_detects_the_information_first(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + devices = det.detect_usage() + + assert [dev.uuid for dev in devices] == ["PPU-0"] + assert devices[0].name == "T-Head PPU" + assert devices[0].memory == 65536 + assert devices[0].cores_utilization == 33 + + +def test_detect_composes_both_queries(detector): + det, _ = detector(_Card(uuid="PPU-0")) + + dev = det.detect()[0] + + assert dev.name == "T-Head PPU" + assert dev.memory == 65536 + assert dev.power == 350 + assert dev.cores_utilization == 33 + assert dev.memory_used == 1024 + assert dev.memory_utilization == _CARD_MEMORY_UTILIZATION + assert dev.temperature == 55 + assert dev.power_used == 150 + + +# --------------------------------------------------------------------------- # +# memory_status, which both queries own. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.usefixtures("health_check") +def test_detect_usage_bounds_a_faulty_instance_to_itself(detector): + # One instance refusing its UUID used to abort the whole card's MIG loop, so + # every later instance kept the inventory's defaults -- 0 % and 0 MiB, i.e. + # reported idle while it may be running a workload. + before_util, after_util = 42.0, 17.0 + det, _ = detector( + _Card( + uuid="PPU-0", + instances=[ + _Instance(uuid="PPU-0-MIG-0", sm_util=before_util), + _Instance(uuid="PPU-0-MIG-1", gpu_instance_id=1, uuid_readable=False), + _Instance(uuid="PPU-0-MIG-2", gpu_instance_id=2, sm_util=after_util), + ], + ), + ) + + mig_devs = det.detect()[0].appendix["mig_devices"] + + by_uuid = {m["uuid"]: m for m in mig_devs} + assert by_uuid["PPU-0-MIG-0"]["cores_utilization"] == before_util + # The instance past the faulty one is still enumerated, with its own reading. + assert by_uuid["PPU-0-MIG-2"]["cores_utilization"] == after_util + + +@pytest.mark.usefixtures("health_check") +def test_detect_keeps_the_memory_status_through_the_merge(detector): + det, _ = detector( + _Card(uuid="PPU-0", instances=[_Instance(uuid="PPU-0-MIG-0")]), + ) + + # merge_devices_usage overwrites memory_status along with the other five + # usage fields, so a usage query that did not re-read the health would wipe + # the information query's verdict back to the UNKNOWN default. The health + # check has to be switched on for this to pin anything: with it off both + # queries answer HEALTHY without a driver call, and the assertion would hold + # whether or not the usage query re-read health at all. + dev = det.detect()[0] + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + assert ( + dev.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.HEALTHY + ) + + +@pytest.mark.usefixtures("health_check") +def test_detect_reports_an_uncorrectable_ecc_error(detector): + det, _ = detector( + _Card( + uuid="PPU-0", + ecc_errors=3, + instances=[_Instance(uuid="PPU-0-MIG-0", ecc_errors=1)], + ), + ) + + # Both queries report the health, mirroring the operator, which flags + # Unhealthy from DetectAccelerator and MonitorAccelerator alike. + info = det.detect_info()[0] + assert info.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert ( + info.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.UNHEALTHY + ) + dev = det.detect()[0] + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert ( + dev.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.UNHEALTHY + ) + + +# --------------------------------------------------------------------------- # +# CDI: /dev/alixpu_ppu{N} is named after the enumeration index. # +# --------------------------------------------------------------------------- # + + +def _fake_node_factory(seen_paths: list[str], minors: dict[str, int]): + """ + Stand in for device_to_cdi_device_node, giving each node a kernel minor. + + The minor is what proves an ordinal-named node reaches the card whose record + names that minor, so a fake that cannot carry one cannot exercise the check. + """ + + def _fake_device_node(path): + seen_paths.append(path) + return SimpleNamespace(path=path, minor=minors.get(path, 0)) + + return _fake_device_node + + +def test_cdi_device_node_path_is_named_after_the_enumeration_index(monkeypatch): + seen_paths: list[str] = [] + + monkeypatch.setattr( + cdi_thead, + "device_to_cdi_device_node", + # The node the ordinal names carries the minor the record states, which + # is the host this fixture reproduces. + _fake_node_factory(seen_paths, {"/dev/alixpu_ppu0": 7}), + ) + + devices = [ + Device( + manufacturer=ManufacturerEnum.THEAD, + index=0, + name="T-Head PPU", + uuid="PPU-0", + memory=65536, + appendix={"bdf": "0000:01:00.0", "minor_number": 7}, + ), + Device( + manufacturer=ManufacturerEnum.THEAD, + index=1, + name="T-Head PPU", + uuid="PPU-1", + memory=65536, + appendix={"bdf": "0000:02:00.0"}, + ), + ] + + config = THeadGenerator().generate(devices) + + assert config is not None + # Unlike /dev/nvidia{N} and /dev/iluvatar{N}, the T-Head node is named after + # the card ordinal and not after the driver's minor number, which the + # operator records purely to PROVE a node addresses the card it describes. + # So the appendix number does not name the node: a card carrying one is + # addressed by its enumeration index all the same, as is one carrying none. + assert "/dev/alixpu_ppu0" in seen_paths + assert "/dev/alixpu_ppu1" in seen_paths + assert "/dev/alixpu_ppu7" not in seen_paths + + +def test_cdi_refuses_a_node_carrying_another_cards_minor(monkeypatch): + # The ordinal names the node and the recorded minor proves that name reaches + # the card the record describes. On the operator's measured host a path built + # from the record lands on the NEIGHBOURING accelerator silently, which is + # the failure this refuses: card 0's record says minor 7, but the node its + # ordinal names carries 8, so that ordinal is addressing another card. + seen_paths: list[str] = [] + monkeypatch.setattr( + cdi_thead, + "device_to_cdi_device_node", + _fake_node_factory(seen_paths, {"/dev/alixpu_ppu0": 8}), + ) + + devices = [ + Device( + manufacturer=ManufacturerEnum.THEAD, + index=0, + name="T-Head PPU", + uuid="PPU-0", + memory=65536, + appendix={"bdf": "0000:01:00.0", "minor_number": 7}, + ), + ] + + config = THeadGenerator().generate(devices) + + # The path was built and then rejected, so no device survives to be handed + # over -- a missing accelerator, never someone else's. + assert "/dev/alixpu_ppu0" in seen_paths + assert config is None