diff --git a/doc/changelog.d/169.fixed.md b/doc/changelog.d/169.fixed.md new file mode 100644 index 00000000..6c94683c --- /dev/null +++ b/doc/changelog.d/169.fixed.md @@ -0,0 +1 @@ +Normalize VTKHDF long arrays to fixed width for the wasm client diff --git a/src/ansys/visor/viewer/vtk/io/client_dataset_normalizer.py b/src/ansys/visor/viewer/vtk/io/client_dataset_normalizer.py new file mode 100644 index 00000000..edb6f44b --- /dev/null +++ b/src/ansys/visor/viewer/vtk/io/client_dataset_normalizer.py @@ -0,0 +1,158 @@ +"""Normalize platform-width VTK arrays before a dataset is served to the wasm client. + +WORKAROUND for an upstream VTK serialization defect: ``vtkObjectManager`` +serializes ``vtkLongArray`` / ``vtkUnsignedLongArray`` by class name only. +On an LP64 server (Linux, macOS) ``long`` is 8 bytes; in the wasm32 +vtkWebAssembly client it is 4 bytes, so the client rebuilds those arrays +with the wrong element width and reads every 64-bit value as two 32-bit +halves. ``vtkHDFReader`` returns 64-bit integer data, including cell +connectivity and offsets, as ``vtkLongArray`` on LP64 hosts, so a dataset +read back from VTKHDF (for example a ``load_state`` snapshot) renders empty +or crashes the client with ``std::bad_alloc``. + +Scope: applied only to datasets Visor itself reads from VTKHDF, in +``file_to_dataset``. Those objects are owned by Visor, so they are converted +in place. In-memory datasets supplied by a caller are deliberately not +converted (that would mutate objects the caller still holds); long-backed +in-memory data remains affected by the upstream defect. + +Remove this module, and its call in ``file_to_dataset``, once the VTK version +Visor depends on serializes platform-width arrays with a fixed element width. +""" + +from vtkmodules.vtkCommonCore import ( + vtkAbstractArray, + vtkDataArray, + vtkTypeInt64Array, + vtkTypeUInt64Array, +) +from vtkmodules.vtkCommonDataModel import ( + vtkCellArray, + vtkCompositeDataSet, + vtkDataObject, + vtkDataSet, + vtkDataSetAttributes, + vtkFieldData, + vtkPolyData, + vtkUnstructuredGrid, +) + +# Matched by exact class name, not isinstance: whether the fixed-width +# classes derive from the platform-width ones depends on the VTK build. +_FIXED_WIDTH_REPLACEMENTS = { + "vtkLongArray": vtkTypeInt64Array, + "vtkUnsignedLongArray": vtkTypeUInt64Array, +} + + +class ClientDatasetNormalizer: + """Rewrites platform-width VTK array classes into fixed-width ones before a dataset reaches the wasm client. + + Temporary workaround for the upstream VTK serialization defect described + in this module's docstring; remove once that defect is fixed upstream. + """ + + @staticmethod + def normalize_dataset_for_client(dataset: vtkDataObject) -> int: + """Normalize every leaf of *dataset* in place; return the number of arrays replaced (0 means nothing changed). + + Values, array names, array order and active-attribute roles are + preserved; only the array classes change. Objects that are neither a + ``vtkDataSet`` nor a ``vtkCompositeDataSet`` are left untouched. + """ + if isinstance(dataset, vtkCompositeDataSet): + replaced = ClientDatasetNormalizer._normalize_field_data(dataset.GetFieldData()) + iterator = dataset.NewIterator() + iterator.SkipEmptyNodesOn() + iterator.InitTraversal() + while not iterator.IsDoneWithTraversal(): + leaf = iterator.GetCurrentDataObject() + if isinstance(leaf, vtkDataSet): + replaced += ClientDatasetNormalizer._normalize_leaf(leaf) + iterator.GoToNextItem() + return replaced + if isinstance(dataset, vtkDataSet): + return ClientDatasetNormalizer._normalize_leaf(dataset) + return 0 + + @staticmethod + def _normalize_leaf(leaf: vtkDataSet) -> int: + """Normalize one non-composite dataset's cell arrays and its point, cell and field data.""" + replaced = sum( + ClientDatasetNormalizer._normalize_cell_array(cells) + for cells in ClientDatasetNormalizer._cell_arrays_of(leaf) + ) + for field_data in (leaf.GetPointData(), leaf.GetCellData(), leaf.GetFieldData()): + replaced += ClientDatasetNormalizer._normalize_field_data(field_data) + if replaced: + leaf.Modified() + return replaced + + @staticmethod + def _cell_arrays_of(leaf: vtkDataSet) -> list[vtkCellArray]: + """Return the cell arrays a leaf carries: verts/lines/polys/strips for polydata, cells for an unstructured grid.""" + if isinstance(leaf, vtkPolyData): + candidates = [leaf.GetVerts(), leaf.GetLines(), leaf.GetPolys(), leaf.GetStrips()] + elif isinstance(leaf, vtkUnstructuredGrid): + candidates = [leaf.GetCells()] + else: + candidates = [] + return [cells for cells in candidates if cells is not None] + + @staticmethod + def _normalize_cell_array(cells: vtkCellArray) -> bool: + """Re-seat generic-storage offsets and connectivity as vtkTypeInt64Array; True if changed.""" + if not cells.IsStorageGeneric(): + return False + offsets = vtkTypeInt64Array() + offsets.DeepCopy(cells.GetOffsetsArray()) + connectivity = vtkTypeInt64Array() + connectivity.DeepCopy(cells.GetConnectivityArray()) + cells.SetData(offsets, connectivity) + return True + + @staticmethod + def _normalize_field_data(field_data: vtkFieldData | None) -> int: + """Replace vtkLongArray/vtkUnsignedLongArray with fixed-width copies, keeping array order, names and active-attribute roles.""" + if field_data is None: + return 0 + arrays = [field_data.GetAbstractArray(i) for i in range(field_data.GetNumberOfArrays())] + replacements = [ClientDatasetNormalizer._fixed_width_copy(array) for array in arrays] + changed = [i for i, replacement in enumerate(replacements) if replacement is not None] + if not changed: + return 0 + + # Active-attribute roles, as (attribute type, array position). + roles: list[tuple[int, int]] = [] + if isinstance(field_data, vtkDataSetAttributes): + for attribute_type in range(vtkDataSetAttributes.NUM_ATTRIBUTES): + active = field_data.GetAbstractAttribute(attribute_type) + for position, array in enumerate(arrays): + if active is not None and active is array: + roles.append((attribute_type, position)) + + # vtkFieldData cannot replace an array in place, so the tail from the + # first changed position is removed and re-appended in order. + first = changed[0] + for position in reversed(range(first, len(arrays))): + field_data.RemoveArray(position) + for position in range(first, len(arrays)): + replacement = replacements[position] + field_data.AddArray(replacement if replacement is not None else arrays[position]) + for attribute_type, position in roles: + field_data.SetActiveAttribute(position, attribute_type) + return len(changed) + + @staticmethod + def _fixed_width_copy(array: vtkAbstractArray | None) -> vtkDataArray | None: + """Return a vtkTypeInt64Array/vtkTypeUInt64Array deep copy of a long array, or None if no conversion is needed.""" + if array is None: + return None + replacement_class = _FIXED_WIDTH_REPLACEMENTS.get(array.GetClassName()) + if replacement_class is None: + return None + replacement = replacement_class() + replacement.DeepCopy(array) + replacement.SetName(array.GetName()) + return replacement + diff --git a/src/ansys/visor/viewer/vtk/io/file_to_dataset.py b/src/ansys/visor/viewer/vtk/io/file_to_dataset.py index a8d94fd5..500121f7 100644 --- a/src/ansys/visor/viewer/vtk/io/file_to_dataset.py +++ b/src/ansys/visor/viewer/vtk/io/file_to_dataset.py @@ -12,6 +12,7 @@ ) from ansys.visor.viewer.core.visor_logging import VisorDefaultLogger +from ansys.visor.viewer.vtk.io.client_dataset_normalizer import ClientDatasetNormalizer """Logging configuration""" logger = VisorDefaultLogger(__name__) @@ -70,6 +71,20 @@ def file_to_dataset(file_path: str) -> vtkDataSet | vtkCompositeDataSet: # Release the reader and its file handle del reader + if is_hdf: + # WORKAROUND (upstream VTK defect, see ClientDatasetNormalizer): + # vtkHDFReader returns 64-bit integers as platform-width long + # arrays on LP64, which the wasm32 client misreads. *copy* is + # owned by Visor, so converting it in place is safe. Remove this + # block with ClientDatasetNormalizer once fixed upstream. + replaced = ClientDatasetNormalizer.normalize_dataset_for_client(copy) + if replaced: + logger.info( + "Normalized %d array(s) of VTKHDF dataset '%s' to fixed-width types for the client.", + replaced, + file_path, + ) + logger.debug(f"Finished creating dataset from file: {file_path}") return copy except RuntimeError as e: diff --git a/tests/e2e/regressions/test_save_load_state.py b/tests/e2e/regressions/test_save_load_state.py index ea9b5c27..4db8b805 100644 --- a/tests/e2e/regressions/test_save_load_state.py +++ b/tests/e2e/regressions/test_save_load_state.py @@ -11,7 +11,6 @@ """ import json -import sys from pathlib import Path import pytest @@ -41,6 +40,21 @@ def _save_state_sync(visor, state_dir: str, force_snapshot: bool = False) -> Non runner.run_coroutine_block(visor.save_state(state_dir), False) +def _served_connectivity_states(visor) -> list[dict]: + """Return the serialized state of each pipeline's polygon connectivity array. + + This is the state the wasm client receives. A ``vtkLongArray`` here is + misread by the wasm32 client on LP64 servers (#122). + """ + renderer = visor._scene._renderer + object_manager = renderer._object_manager + states = [] + for pipe in renderer._pipelines.values(): + connectivity = pipe.mapper.GetInput().GetPolys().GetConnectivityArray() + states.append(json.loads(object_manager.GetState(object_manager.GetId(connectivity)))) + return states + + @pytest.mark.regression class TestSaveLoadState: """Regression tests for save/load state functionality.""" @@ -143,14 +157,6 @@ def test_save_load_state_file_content_valid(self, visor_server, page, tmp_path): f"Dataset '{name}' has empty serialized_dataset_path" ) - @pytest.mark.xfail( - sys.platform != "win32", - run=False, - reason="#122: load_state into an empty scene does not render on " - "Linux, and the shared server is not recoverable afterwards, " - "so stopping this test from running there. The post-load checks " - "pass on the failing state, which is why this was not caught earlier." - ) def test_load_state_into_empty_scene(self, visor_server, page, tmp_path): """Loading state into an empty scene should restore datasets from snapshots. @@ -160,6 +166,9 @@ def test_load_state_into_empty_scene(self, visor_server, page, tmp_path): state_dir = str(tmp_path / "state") Path(state_dir).mkdir() + console_errors = [] + page.on("console", lambda msg: console_errors.append(msg.text) if msg.type == "error" else None) + # Frontend must be connected for save_state goto_and_wait(page, visor_server.url) wait_for_canvas(page) @@ -188,3 +197,12 @@ def test_load_state_into_empty_scene(self, visor_server, page, tmp_path): wait_for_canvas(page) wait_for_webgl(page, timeout_ms=15_000) assert_canvas_alive(page, "after load_state into empty scene") + + # The canvas checks above also pass on an empty canvas (#122), so check + # that geometry reached the client in a type wasm32 reads correctly. + served = _served_connectivity_states(visor_server) + assert len(served) > 0, "No pipelines after load_state into empty scene" + for state in served: + assert state["ClassName"] not in ("vtkLongArray", "vtkUnsignedLongArray"), state + assert state["NumberOfTuples"] > 0, state + assert [e for e in console_errors if "bad_alloc" in e] == [] diff --git a/tests/unit/vtk/io/test_client_dataset_normalizer.py b/tests/unit/vtk/io/test_client_dataset_normalizer.py new file mode 100644 index 00000000..35d750f2 --- /dev/null +++ b/tests/unit/vtk/io/test_client_dataset_normalizer.py @@ -0,0 +1,147 @@ +"""Unit tests for ClientDatasetNormalizer. + +Inputs are hand-built ``vtkLongArray`` / ``vtkUnsignedLongArray`` datasets, so +these run on any OS: a ``vtkLongArray``-backed ``vtkCellArray`` uses generic +storage everywhere, which is what ``vtkHDFReader`` produces on LP64 hosts. +""" + +from vtkmodules.vtkCommonCore import ( + vtkFloatArray, + vtkLongArray, + vtkPoints, + vtkTypeInt64Array, + vtkUnsignedCharArray, + vtkUnsignedLongArray, +) +from vtkmodules.vtkCommonDataModel import ( + VTK_TRIANGLE, + vtkCellArray, + vtkMultiBlockDataSet, + vtkPolyData, + vtkUnstructuredGrid, +) + +from ansys.visor.viewer.vtk.io.client_dataset_normalizer import ClientDatasetNormalizer + + +def _array(cls, name, values): + array = cls() + array.SetName(name) + for value in values: + array.InsertNextValue(value) + return array + + +def _triangle_points() -> vtkPoints: + points = vtkPoints() + for p in ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 1.0, 0.0)): + points.InsertNextPoint(p) + return points + + +def _cells(cls) -> vtkCellArray: + cells = vtkCellArray() + cells.SetData(_array(cls, None, (0, 3)), _array(cls, None, (0, 1, 2))) + return cells + + +def _triangle_polydata(cell_array_class: type = vtkLongArray) -> vtkPolyData: + poly = vtkPolyData() + poly.SetPoints(_triangle_points()) + poly.SetPolys(_cells(cell_array_class)) + return poly + + +def _triangle_grid() -> vtkUnstructuredGrid: + grid = vtkUnstructuredGrid() + grid.SetPoints(_triangle_points()) + types = vtkUnsignedCharArray() + types.InsertNextValue(VTK_TRIANGLE) + grid.SetCells(types, _cells(vtkLongArray)) + return grid + + +def _values(array) -> list: + return [array.GetValue(i) for i in range(array.GetNumberOfValues())] + + +def test_generic_polydata_cells_become_int64_storage(): + """Generic-storage cells are re-seated as vtkTypeInt64Array with the same values.""" + poly = _triangle_polydata() + + ClientDatasetNormalizer.normalize_dataset_for_client(poly) + + cells = poly.GetPolys() + assert cells.IsStorageGeneric() is False + assert cells.GetConnectivityArray().GetClassName() == "vtkTypeInt64Array" + assert cells.GetOffsetsArray().GetClassName() == "vtkTypeInt64Array" + assert _values(cells.GetConnectivityArray()) == [0, 1, 2] + assert _values(cells.GetOffsetsArray()) == [0, 3] + + +def test_long_point_cell_and_field_arrays_become_fixed_width(): + """Long point, cell and field arrays become fixed-width with name and values kept.""" + poly = _triangle_polydata() + poly.GetPointData().AddArray(_array(vtkLongArray, "pid", (7, 8, 9))) + poly.GetCellData().AddArray(_array(vtkUnsignedLongArray, "cid", (5,))) + poly.GetFieldData().AddArray(_array(vtkLongArray, "fid", (11, 12))) + + ClientDatasetNormalizer.normalize_dataset_for_client(poly) + + pid = poly.GetPointData().GetArray("pid") + cid = poly.GetCellData().GetArray("cid") + fid = poly.GetFieldData().GetArray("fid") + assert (pid.GetClassName(), _values(pid)) == ("vtkTypeInt64Array", [7, 8, 9]) + assert (cid.GetClassName(), _values(cid)) == ("vtkTypeUInt64Array", [5]) + assert (fid.GetClassName(), _values(fid)) == ("vtkTypeInt64Array", [11, 12]) + + +def test_array_order_and_active_scalars_are_kept(): + """Replacing a middle array keeps array order and its active-scalars role.""" + poly = _triangle_polydata() + point_data = poly.GetPointData() + point_data.AddArray(_array(vtkFloatArray, "a", (1.0, 2.0, 3.0))) + point_data.AddArray(_array(vtkLongArray, "b", (4, 5, 6))) + point_data.AddArray(_array(vtkFloatArray, "c", (7.0, 8.0, 9.0))) + point_data.SetActiveScalars("b") + + ClientDatasetNormalizer.normalize_dataset_for_client(poly) + + names = [point_data.GetArrayName(i) for i in range(point_data.GetNumberOfArrays())] + assert names == ["a", "b", "c"] + assert point_data.GetScalars().GetName() == "b" + assert point_data.GetScalars().GetClassName() == "vtkTypeInt64Array" + + +def test_composite_leaves_are_normalized(): + """Leaves of a nested multiblock (polydata and unstructured grid) are both normalized.""" + inner = vtkMultiBlockDataSet() + inner.SetNumberOfBlocks(1) + inner.SetBlock(0, _triangle_polydata()) + outer = vtkMultiBlockDataSet() + outer.SetNumberOfBlocks(2) + outer.SetBlock(0, inner) + outer.SetBlock(1, _triangle_grid()) + + replaced = ClientDatasetNormalizer.normalize_dataset_for_client(outer) + + poly = outer.GetBlock(0).GetBlock(0) + grid = outer.GetBlock(1) + assert poly.GetPolys().GetConnectivityArray().GetClassName() == "vtkTypeInt64Array" + assert grid.GetCells().GetConnectivityArray().GetClassName() == "vtkTypeInt64Array" + assert replaced == 2 + + +def test_fixed_width_dataset_is_left_untouched(): + """A dataset with nothing to convert is a no-op: count 0, same array objects.""" + poly = _triangle_polydata(cell_array_class=vtkTypeInt64Array) + poly.GetPointData().AddArray(_array(vtkFloatArray, "a", (1.0, 2.0, 3.0))) + connectivity = poly.GetPolys().GetConnectivityArray() + scalars = poly.GetPointData().GetArray("a") + + replaced = ClientDatasetNormalizer.normalize_dataset_for_client(poly) + + assert replaced == 0 + assert poly.GetPolys().GetConnectivityArray() is connectivity + assert poly.GetPointData().GetArray("a") is scalars + diff --git a/tests/unit/vtk/io/test_file_to_dataset.py b/tests/unit/vtk/io/test_file_to_dataset.py index 0ce7f588..759ba566 100644 --- a/tests/unit/vtk/io/test_file_to_dataset.py +++ b/tests/unit/vtk/io/test_file_to_dataset.py @@ -1,7 +1,8 @@ from unittest.mock import MagicMock, patch import pytest -from vtkmodules.vtkCommonDataModel import vtkPolyData +from vtkmodules.vtkCommonCore import vtkLongArray, vtkPoints +from vtkmodules.vtkCommonDataModel import vtkCellArray, vtkPolyData from ansys.visor.viewer.vtk.io.file_to_dataset import file_to_dataset @@ -140,3 +141,49 @@ def test_file_to_dataset_vtkhdf_uses_hdf_reader_not_xml_reader(): mock_xml_vtu.assert_not_called() mock_xml_vtm.assert_not_called() + +# ------------------------------------------------------------------ # +# wasm32 long-array workaround (ClientDatasetNormalizer) +# ------------------------------------------------------------------ # + +def _long_backed_triangle() -> vtkPolyData: + """Triangle whose cells and point data are vtkLongArray, as vtkHDFReader returns them on LP64.""" + def long_array(name, values): + array = vtkLongArray() + array.SetName(name) + for value in values: + array.InsertNextValue(value) + return array + + points = vtkPoints() + for point in ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 1.0, 0.0)): + points.InsertNextPoint(point) + cells = vtkCellArray() + cells.SetData(long_array(None, (0, 3)), long_array(None, (0, 1, 2))) + dataset = vtkPolyData() + dataset.SetPoints(points) + dataset.SetPolys(cells) + dataset.GetPointData().AddArray(long_array("pid", (7, 8, 9))) + return dataset + + +@pytest.mark.parametrize("ext,reader_cls,expected_class", [ + ("vtkhdf", "vtkHDFReader", "vtkTypeInt64Array"), + ("vtp", "vtkXMLPolyDataReader", "vtkLongArray"), +]) +def test_file_to_dataset_normalizes_long_arrays_only_for_vtkhdf(ext, reader_cls, expected_class): + """VTKHDF output is converted to fixed-width arrays; other formats are returned as read.""" + with patch("os.path.exists", return_value=True), \ + patch("os.path.basename", return_value=f"model.{ext}"), \ + patch("os.path.splitext", return_value=("model", f".{ext}")), \ + patch(f"ansys.visor.viewer.vtk.io.file_to_dataset.{reader_cls}") as mock_reader_cls: + mock_reader = MagicMock() + mock_reader.GetOutput.return_value = _long_backed_triangle() + mock_reader_cls.return_value = mock_reader + result = file_to_dataset(f"/tmp/model.{ext}") + + connectivity = result.GetPolys().GetConnectivityArray() + pid = result.GetPointData().GetArray("pid") + assert (connectivity.GetClassName(), pid.GetClassName()) == (expected_class, expected_class) + assert [connectivity.GetValue(i) for i in range(3)] == [0, 1, 2] + assert [pid.GetValue(i) for i in range(3)] == [7, 8, 9]