add parents to Groups and Datasets objects

src/hdfmap/eval_functions.py

@@ -19,7 +19,7 @@
 GLOBALS_NAMELIST = asteval.make_symbol_table(use_numpy=True).keys()
 DEFAULT: typing.Any = np.array('--')  # default return in eval
 SEP = '/'  # HDF path separator
-OMIT = '/value'  # omit this name in paths when determining identifier
+OMIT = ['/value', '/data']  # omit these names in paths when determining identifier
 logger = create_logger(__name__)
 # regex patterns
 re_special_characters = re.compile(r'\W')  # finds all special non-alphanumberic characters
@@ -38,7 +38,7 @@ def generate_identifier(hdf_path: str | bytes) -> str:
     """
     Generate a valid python identifier from a hdf dataset path or other string
      - Decodes to ascii
-     - omits '/value'
+     - omits '/value' and '/data'
      - splits by path separator (/) and takes final element
      - converts special characters to '_'
      - removes replication of strings separated by '_'
@@ -52,12 +52,12 @@ def generate_identifier(hdf_path: str | bytes) -> str:
     """
     if hasattr(hdf_path, 'decode'):  # Byte string
         hdf_path = hdf_path.decode('ascii')
-    if hdf_path.endswith(OMIT):
-        hdf_path = hdf_path[:-len(OMIT)]  # omit 'value'
-    substrings = hdf_path.split(SEP)
-    name = expression_safe_name(substrings[-1])
+    for omit in OMIT:
+        hdf_path = hdf_path.removesuffix(omit)  # omit 'value' or 'data'
+    substring = hdf_path.split(SEP)[-1]
     # remove replication (handles local_names 'name.name' convention)
-    return '_'.join(dict.fromkeys(name.split('_')))
+    substring = '.'.join(dict.fromkeys(substring.split('.')))
+    return expression_safe_name(substring)
 
 
 def build_hdf_path(*args: str | bytes) -> str:
@@ -150,6 +150,7 @@ def dataset2data(dataset: h5py.Dataset, index: int | slice = (), direct_load=Fal
         return dataset[index]
     if np.issubdtype(dataset, np.number):
         logger.debug(f"Dataset {repr(dataset)} is numeric, return numpy array")
+        # note that squeeze returns float if np.float64 is given.
         return np.squeeze(dataset[index])  # numeric np.ndarray
     try:
         # str integers will be cast as timestamps (years), capture as int

src/hdfmap/hdfmap_class.py

@@ -29,6 +29,8 @@ class Group(typing.NamedTuple):
     name: str
     attrs: dict
     datasets: list[str]
+    parent: "Group | None"
+    default: bool
 
 
 class Dataset(typing.NamedTuple):
@@ -37,6 +39,7 @@ class Dataset(typing.NamedTuple):
     size: int
     shape: tuple[int]
     attrs: dict
+    parent: Group
 
 
 def generate_alt_name(hdf_dataset: h5py.Dataset) -> str | None:
@@ -273,14 +276,18 @@ def _store_class(self, name, path):
 
     def _store_group(self, hdf_group: h5py.Group, path: str, name: str):
 
-        nx_class = hdf_group.attrs.get('NX_class', default='Group')
-        if hasattr(nx_class, 'decode'):
-            nx_class = nx_class.decode()
+        parent = self.groups.get(hdf_group.parent.name, None)
+        attrs = attrs2dict(hdf_group)
+        parent_attrs = parent.attrs if parent else {}
+        nx_class = attrs.get('NX_class', 'Group')
+        nx_default = parent_attrs.get('default', None) == name
         self.groups[path] = Group(
-            nx_class,
-            name,
-            attrs2dict(hdf_group),
-            [key for key, item in hdf_group.items() if isinstance(item, h5py.Dataset)]
+            nx_class=nx_class,
+            name=name,
+            attrs=attrs,
+            datasets=[key for key, item in hdf_group.items() if isinstance(item, h5py.Dataset)],
+            parent=parent,
+            default=nx_default
         )
         self._store_class(name, path)
         self._store_class(nx_class, path)
@@ -303,23 +310,30 @@ def _store_dataset(self, hdf_dataset: h5py.Dataset, hdf_path: str, name: str):
             size=hdf_dataset.size,
             shape=hdf_dataset.shape,
             attrs=attrs2dict(hdf_dataset),
+            parent=group,
         )
-        if is_image(hdf_dataset.shape):
-            self.image_data[name] = hdf_path
-            self.image_data[group_name] = hdf_path
-            self.arrays.update(names)
-            logger.debug(f"{hdf_path}  HDFDataset: image_data & array {name, hdf_dataset.size, hdf_dataset.shape}")
-        elif hdf_dataset.ndim > 0:
-            self.arrays.update(names)
+        if hdf_dataset.ndim > 0:
+            if is_image(hdf_dataset.shape):
+                logger.debug(f"{hdf_path}  HDFDataset: image_data {name, hdf_dataset.size, hdf_dataset.shape}")
+                self.image_data[name] = hdf_path
+                self.image_data[group_name] = hdf_path
+                if hdf_path.endswith('/data'):
+                    names['data'] = hdf_path  # add data name back as removed by generate_identifiers
+            array_names = {
+                name: path for name, path in names.items()
+                if group.default or name not in self.arrays
+            }
+            self.arrays.update(array_names)
             logger.debug(f"{hdf_path}  HDFDataset: array {name, hdf_dataset.size, hdf_dataset.shape}")
         else:
             self.values.update(names)
             if alt_name:
+                # metadata only stored for single value entries with alt_name defined
                 self.metadata[alt_name] = hdf_path
             logger.debug(f"{hdf_path}  HDFDataset: value")
 
     def _populate(self, hdf_group: h5py.Group, root: str = '',
-                  recursive: bool = True, groups: list[str] = None):
+                  recursive: bool = True, groups: list[str] | None = None):
         """
         populate HdfMap dictionary's using recursive method
         :param hdf_group: HDF group object, from HDF File
@@ -538,8 +552,8 @@ def generate_scannables(self, array_size):
         # create combined dict, scannables and arrays overwrite values with same name
         # self.generate_combined()
 
-    def generate_scannables_from_group(self, hdf_group: h5py.Group, group_path: str = None,
-                                       dataset_names: list[str] = None):
+    def generate_scannables_from_group(self, hdf_group: h5py.Group, group_path: str | None = None,
+                                       dataset_names: list[str] | None = None):
         """
         Generate scannables list from a specific group, using the first item to define array size
         :param hdf_group: h5py.Group
@@ -581,50 +595,25 @@ def generate_scannables_from_names(self, names: list[str]):
         logger.debug(f"Scannables from names: {array_names}")
         array_size = self.datasets[self.arrays[array_names[0]]].size
         self.scannables = {
-            name: self.arrays[name] for name in array_names if self.datasets[self.arrays[name]].size == array_size
+            self.datasets[self.arrays[name]].name: self.arrays[name]
+            for name in array_names
+            if self.datasets[self.arrays[name]].size == array_size
         }
         # self.generate_combined()
 
-    def first_last_scannables(self, first_names: list[str] = (),
-                              last_names: list[str] = (),
-                              alt_names: dict[str, list[str]] | None = None) -> tuple[dict[str, str], dict[str, str]]:
+    def first_last_scannables(self, last_n: int = 1) -> tuple[dict[str, str], dict[str, str]]:
         """
-        Returns default names from scannables
-            output first_names returns dict of N names, where N is the number of dimensions in scannable shape
-                if fewer axes_names are provided than required, use the first items of scannables instead
-            output signal_names returns the last dict item in the list of scannables + signal_names
-
-        :param first_names: list of names of plottable axes in scannables
-        :param last_names: list of names of plottable values in scannables
-        :param alt_names: dict of alternative names for each plottable value
+        Returns default names from scannables based on order
+            output first N scannables, where N is the number of dimensions of scannables_shape()
+            Output last N scannables, where N is last_n
+
+        :param last_n: return last N scannables
         :return {first_names: path}, {last_names: path}
         """
-        if alt_names is None:
-            alt_names = {}
-        list_names = list(first_names) + list(self.scannables.keys()) + list(last_names)
-        # check names are in scannables
-        warnings = []
-        all_names = []
-        for name in list_names:
-            if name in all_names:
-                continue
-            elif name in self.scannables:
-                all_names.append(name)
-            elif name in alt_names:
-                alt_name = next((alt for alt in alt_names[name] if alt in self.scannables), None)
-                if alt_name:
-                    all_names.append(alt_name)
-                else:
-                    warnings.append(name)
-            else:
-                warnings.append(name)
-
-        for name in warnings:
-            logger.warning(f"name: '{name}' not in scannables")
-        # return correct number of values from start and end
-        ndims = len(self.scannables_shape())
-        first = {name: self.scannables[name] for name in all_names[:ndims]}
-        last = {name: self.scannables[name] for name in all_names[-(len(last_names) or 1):]}
+        scannable_names = list(self.scannables)
+        first_n = len(self.scannables_shape())
+        first = {name: self.scannables[name] for name in scannable_names[:first_n]}
+        last = {name: self.scannables[name] for name in scannable_names[::-1][:last_n]}
         return first, last
 
     def get_path(self, name_or_path):
@@ -805,7 +794,8 @@ def get_image_shape(self) -> tuple[int, int]:
         """Return the scan shape of the detector dataset"""
         path = self.get_image_path()
         if path in self.datasets:
-            return self.datasets[path].shape[-2:]
+            i, j = self.datasets[path].shape[-2:]
+            return i, j
         return 0, 0
 
     def get_image_index(self, index: int) -> tuple[int, ...]:
@@ -864,7 +854,7 @@ def generate_json_str(self) -> str:
     "---------------------- FILE READERS --------------------"
     "--------------------------------------------------------"
 
-    def load_hdf(self, filename: str | None = None, name_or_path: str = None, **kwargs) -> h5py.File | h5py.Dataset:
+    def load_hdf(self, filename: str | None = None, name_or_path: str | None = None, **kwargs) -> h5py.File | h5py.Dataset:
         """
         Load hdf file or hdf dataset in open state
         :param filename: str filename of hdf file, or None to use self.filename
@@ -911,7 +901,7 @@ def get_string(self, hdf_file: h5py.File, name_or_path: str, index=(), default='
         return default
 
     def get_metadata(self, hdf_file: h5py.File, default=None, direct_load=False,
-                     name_list: list = None, string_output=False, numeric_only=False) -> dict:
+                     name_list: list | None = None, string_output=False, numeric_only=False) -> dict:
         """
         Return metadata dict from file, loading data for each item in the metadata list
         The metadata list is taken from name_list, otherwise self.metadata or self.values
@@ -951,7 +941,7 @@ def get_metadata(self, hdf_file: h5py.File, default=None, direct_load=False,
             }
         return {**extra, **metadata}
 
-    def create_metadata_list(self, hdf_file: h5py.File, default=None, name_list: list = None,
+    def create_metadata_list(self, hdf_file: h5py.File, default=None, name_list: list | None = None,
                              line_separator: str = '\n', value_separator: str = '=') -> str:
         """
         Return a metadata string, using self.get_metadata
@@ -1000,7 +990,9 @@ def get_image(self, hdf_file: h5py.File, index: int | tuple | slice | None = Non
         logger.info(f"image path: {image_path}")
         if image_path and image_path in hdf_file:
             # return hdf_file[image_path][index].squeeze()  # remove trailing dimensions
-            return self.get_data(hdf_file, image_path, index)  # return array or image paths
+            # note that squeeze returns float if np.float64 is given.
+            image_array = np.asarray(self.get_data(hdf_file, image_path, index))
+            return image_array
         return None
 
     def _get_numeric_scannables(self, hdf_file: h5py.File) -> list[tuple[str, str, np.ndarray]]:

src/hdfmap/nexus.py

@@ -168,13 +168,13 @@ def names_from_scan_fields(hdf_file: h5py.File, scan_fields_path: str) -> list[s
 
     scan_fields stores scannables as class_name.dataset_name, return only the dataset_name
 
-    :param hdf_file:
-    :param scan_fields_path:
-    :returns: ['names',]
+    :param hdf_file: h5py.File object, reads 'scan_fields' dataset
+    :param scan_fields_path: str path to the 'scan_fields' dataset
+    :returns: list of names for datasets
     """
     scan_fields_dataset = hdf_file.get(scan_fields_path)
     if scan_fields_dataset:
-        return [name.decode().split('.')[-1] for name in scan_fields_dataset[()]]
+        return [generate_identifier(name) for name in scan_fields_dataset.asstr()[...]]
     return []
 
 
@@ -279,23 +279,68 @@ def _store_default_nexus_paths(self, hdf_file):
             pass
 
     def nexus_default_paths(self) -> tuple[list[str], list[str]]:
-        """Return default axes and signal paths"""
+        """
+        Return default axes and signal paths, as defined by NeXus
+
+        The default axes and signal are defined as attributes @axes and @signal
+        of the @default NXdata within the @default NXentry groups
+        of the NeXus file.
+
+        The number of axes will equal the number of scannable dimensions.
+        There may be more then one signal path, as @auxilliary_signals may
+        contain additional paths, however the first in the list is @signal.
+
+        :return: list of hdf paths for axes, list of hdf paths for signals
+        """
         axes_paths = [self.arrays[axes] for n in range(10) if (axes := f"{NX_AXES}{n}") in self.arrays]
         signal_paths = [self.arrays[signal] for n in range(10) if (signal := f"{NX_SIGNAL}{n}") in self.arrays]
         return axes_paths, signal_paths
 
     def nexus_default_names(self) -> tuple[dict[str, str], dict[str, str]]:
-        """Return name of default axes and signal paths, as defined in scannables"""
+        """
+        Return name of default axes and signal names, as defined by scannables
+
+        The way by which scannables are set will change the output.
+
+        By Default, NeXus default paths will be used, providing they are available in the
+        list of scannables.
+
+        :return: {'axes name: 'hdf_path', ...}, {'signal name: 'hdf_path', ...}
+        """
         axes_paths, signal_paths = self.nexus_default_paths()
         axes_names = [self.datasets[path].name for path in axes_paths]
         signal_names = [self.datasets[path].name for path in signal_paths]
-        # axes_names = [name for path in axes_paths for name in self.datasets[path].names]
-        # signal_names = [name for path in signal_paths for name in self.datasets[path].names]
         alt_names = {
             self.datasets[path].name: self.datasets[path].names
             for path in axes_paths + signal_paths
         }
-        return self.first_last_scannables(axes_names, signal_names, alt_names)
+        scannable_names = list(self.scannables)
+
+        for name in axes_names:
+            if name not in self.scannables:
+                if name in self.arrays and self.datasets[self.arrays[name]].shape == self.scannables_shape():
+                    logger.warning(f"axes '{name}' not found in scannables, appending '{name}' to scannables")
+                    self.scannables[name] = self.arrays[name]
+                else:
+                    raise KeyError(f"axes '{name}' not found in scannables")
+        for n, name in enumerate(signal_names):
+            scannable_name = next(
+                (alt_name for alt_name in alt_names[name] if alt_name in scannable_names),
+                None
+            )
+            if scannable_name is None:
+                if name in self.arrays and self.datasets[self.arrays[name]].shape == self.scannables_shape():
+                    logger.warning(f"signal '{name}' not found in scannables, appending '{name}' to scannables")
+                    self.scannables[name] = self.arrays[name]
+                else:
+                    logger.warning(
+                        f"signal '{name}' not found in scannables, " +
+                        f"switching to '{scannable_names[-(n + 1)]}'"
+                    )
+                    signal_names[n] = scannable_names[-(n + 1)]
+        axes_dict = {name: self.scannables[name] for name in axes_names}
+        signal_dict = {name: self.scannables[name] for name in signal_names}
+        return axes_dict, signal_dict
 
     def generate_scannables_from_nxdata(self, hdf_file: h5py.File, use_auxiliary: bool = True):
         """Generate scannables from default NXdata, using axuiliary_names if available"""
@@ -325,7 +370,6 @@ def generate_scannables_from_scan_fields_or_nxdata(self, hdf_file: h5py.File):
         # find 'scan_fields' to generate scannables list
         if NX_SCANFIELDS in self.arrays:
             scan_fields_path = self.arrays[NX_SCANFIELDS]
-            # scan_fields = hdf_file[scan_fields_path][()]
             scan_fields = names_from_scan_fields(hdf_file, scan_fields_path)
             if scan_fields:
                 logger.info(f"Generating Scannables from NX ScanFields: {scan_fields_path}: {scan_fields}")

tests/test_edge_cases.py

@@ -15,6 +15,7 @@
 def test_edge_cases():
     n = 0
     for chk in CHECK_FILES:
+        print(f"Checking {chk['filename']}")
         assert path.isfile(chk['filename']) is True, f"{chk['filename']} doesn't exist"
         mymap = hdfmap.create_nexus_map(chk['filename'])
         assert isinstance(mymap, hdfmap.NexusMap), f"{chk['filename']} is not NexusMap"
@@ -72,6 +73,7 @@ def test_newer_i16_file():
 
 @only_dls_file_system
 def test_i16_bpm_file():
+    """Tests an i16 scan with bpm images which are returned in the old style as TIFF"""
     filename = '/dls/science/groups/das/ExampleData/hdfmap_tests/i16/1113658.nxs'
     assert path.isfile(filename) is True, f"{filename} doesn't exist"
     mymap = hdfmap.create_nexus_map(filename)
@@ -82,6 +84,33 @@ def test_i16_bpm_file():
     assert int(image) == 11, 'bpm image has wrong value'
 
 
+@only_dls_file_system
+def test_i16_default_signal():
+    mymap = hdfmap.create_nexus_map('/dls/science/groups/das/ExampleData/i16/azimuths/1108750.nxs')
+
+    axes_names, signal_names = mymap.nexus_default_names()
+
+    assert next(iter(axes_names)) == 'eta_fly_fly'
+    assert next(iter(signal_names)) == 'mroi2_sum'
+    assert axes_names == {'eta_fly_fly': '/entry/measurement/eta_fly_fly'}
+    assert len(signal_names) == 6
+    assert signal_names == {
+        'mroi2_sum': '/entry/instrument/mroi2/mroi2_sum',
+        'count_time': '/entry/instrument/merlin/count_time',
+        'merlin_max_val': '/entry/instrument/merlin/merlin_max_val',
+        'merlin_max_x': '/entry/instrument/merlin/merlin_max_x',
+        'merlin_max_y': '/entry/instrument/merlin/merlin_max_y',
+        'merlin_total': '/entry/instrument/merlin/merlin_total'
+    }
+
+
+@only_dls_file_system
+def test_i10_scannables():
+    mymap = hdfmap.create_nexus_map('/dls/science/groups/das/ExampleData/hdfmap_tests/i10/i10-1-28428.nxs')
+    scan_fields = mymap.get_data(mymap.load_hdf(), 'scan_fields')
+    assert len(mymap.scannables) == len(scan_fields)
+
+
 @only_dls_file_system
 def test_i16_burst_mode():
     # See https://github.com/DiamondLightSource/hdfmap/issues/29