From 6d08ded4107ed2425ba1ed1cae37a29e8ac9af3a Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Thu, 27 Aug 2026 12:21:24 +0100 Subject: [PATCH 1/9] add get_default_name Adds a new function to return default dataset or group name ## hdfmap_class.py - Add *HdfMap.get_default_name()* - add test --- src/hdfmap/eval_functions.py | 3 +++ src/hdfmap/hdfmap_class.py | 11 ++++++++++- tests/test_hdfmap_class.py | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index af1d708..600db4c 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -385,6 +385,9 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict :param replace_names: dict of {'variable_name': expression} :return: str expression """ + #TODO: remove hdf from this function, replace with dict of Dataset objects. + #TODO: Move Dataset objects into seperate file that can be improted + # replace names with expressions expression = replace_expression_vars(expression, replace_names) diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index eb58c44..fbd805e 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -621,7 +621,16 @@ def first_last_scannables(self, last_n: int = 1) -> tuple[dict[str, str], dict[s last = {name: self.scannables[name] for name in scannable_names[::-1][:last_n]} return first, last - def get_path(self, name_or_path): + def get_default_name(self, name_or_path: str) -> str | None: + """Return the default name of a dataset or group identifier""" + path = self.get_path(name_or_path) + if path and path in self.datasets: + return self.datasets[path].name + elif path and path in self.groups: + return self.groups[path].name + return None + + def get_path(self, name_or_path) -> str | None: """Return hdf path of object in HdfMap""" if name_or_path in self.datasets or name_or_path in self.groups: return name_or_path diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index ad5237d..b13b8e5 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -68,6 +68,10 @@ def test_get_path(hdf_map): assert hdf_map.get_path('NXdata') == '/entry1/measurement', 'class is wrong' +def test_get_default_name(hdf_map): + assert hdf_map.get_default_name('measurement_sum') == 'sum' + + def test_get_group_path(hdf_map): assert hdf_map.get_group_path('sum') == '/entry1/pil3_100k' From 229ccf185cb81c08751e1e0ad8422caa74277575 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 10:38:18 +0100 Subject: [PATCH 2/9] refactor items ## hdfmap_class.py - move *generate_alt_name()* into eval_functions.py - move Dataset and Group into objects.py --- src/hdfmap/eval_functions.py | 11 ++++++++++ src/hdfmap/hdfmap_class.py | 40 +++++------------------------------- src/hdfmap/objects.py | 28 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 35 deletions(-) create mode 100644 src/hdfmap/objects.py diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index 600db4c..0d6e53e 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -20,6 +20,7 @@ DEFAULT: typing.Any = np.array('--') # default return in eval SEP = '/' # HDF path separator OMIT = ['/value', '/data'] # omit these names in paths when determining identifier +LOCAL_NAME = 'local_name' # dataset attribute name for alt_name (DLS specific) logger = create_logger(__name__) # regex patterns re_special_characters = re.compile(r'\W') # finds all special non-alphanumberic characters @@ -33,6 +34,16 @@ logger.warning("Nexus timestamps are not convertable by datetime.fromisoformat in python version <3.11") +def generate_alt_name(hdf_dataset: h5py.Dataset) -> str | None: + """Generate alt_name of dataset if 'local_name' in attributes""" + if LOCAL_NAME in hdf_dataset.attrs: + alt_name = hdf_dataset.attrs[LOCAL_NAME] + if hasattr(alt_name, 'decode'): + alt_name = alt_name.decode() + return expression_safe_name(alt_name.split('.')[-1]) + return None + + def generate_identifier(hdf_path: str | bytes) -> str: """ Generate a valid python identifier from a hdf dataset path or other string diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index fbd805e..4947718 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -13,47 +13,16 @@ from .logging import create_logger from .eval_functions import (expression_safe_name, extra_hdf_data, eval_hdf, HdfMapInterpreter, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, - DEFAULT, SEP, generate_identifier, build_hdf_path) - + DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) +from .objects import Group, Dataset # parameters -LOCAL_NAME = 'local_name' # dataset attribute name for alt_name IMAGE_DATA = 'IMAGE' # namespace name for default image data # logger logger = create_logger(__name__) -class Group(typing.NamedTuple): - nx_class: str - name: str - attrs: dict - datasets: list[str] - parent: "Group | None" - default: bool - external_file: str | None - - -class Dataset(typing.NamedTuple): - name: str - names: list[str] - size: int - shape: tuple[int] - attrs: dict - parent: Group - external_file: str | None - - -def generate_alt_name(hdf_dataset: h5py.Dataset) -> str | None: - """Generate alt_name of dataset if 'local_name' in attributes""" - if LOCAL_NAME in hdf_dataset.attrs: - alt_name = hdf_dataset.attrs[LOCAL_NAME] - if hasattr(alt_name, 'decode'): - alt_name = alt_name.decode() - return expression_safe_name(alt_name.split('.')[-1]) - return None - - class HdfMap: """ HdfMap object, container for paths of different objects in an HDF file @@ -854,6 +823,7 @@ def generate_ids(self, *names: str, modify_missing: bool = True) -> list[str]: :param modify_missing: if True, modifies names even if they are not in namespace :return: list of axis labels as valid identifiers """ + #TODO: use get_default_names or similar here, or leave it as different return [ generate_identifier(self.combined.get(name, name)) if modify_missing else ( generate_identifier(self.combined[name]) if name in self.combined else name @@ -1104,7 +1074,7 @@ def get_dataholder(self, hdf_file: h5py.File, flatten_scannables: bool = False) def eval(self, hdf_file: h5py.File, expression: str, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None, - raise_errors: bool = True): + raise_errors: bool = True) -> typing.Any: """ Evaluate an expression using the namespace of the hdf file :param hdf_file: h5py.File object @@ -1150,7 +1120,7 @@ def format_hdf(self, hdf_file: h5py.File, expression: str, default=DEFAULT, raise_errors=raise_errors ) - def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None): + def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None) -> HdfMapInterpreter: """ Create an interpreter object for the current file The interpreter is a sub-class of asteval.Interpreter that parses expressions for hdfmap eval patters diff --git a/src/hdfmap/objects.py b/src/hdfmap/objects.py new file mode 100644 index 0000000..6f6428e --- /dev/null +++ b/src/hdfmap/objects.py @@ -0,0 +1,28 @@ +""" +Definition of HdfMap objects for HDF Groups and Datasets + +These objects are a simplification of the HDF objects and won't require +the file to be open to view them, plus they are more easily serialised. +""" + +from typing import NamedTuple + + +class Group(NamedTuple): + nx_class: str + name: str + attrs: dict + datasets: list[str] + parent: "Group | None" + default: bool + external_file: str | None + + +class Dataset(NamedTuple): + name: str + names: list[str] + size: int + shape: tuple[int, ...] + attrs: dict + parent: Group + external_file: str | None From 410c0c94ea983f9c54dcf2b286e9815468930892 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 10:48:59 +0100 Subject: [PATCH 3/9] prepare expression using Datasets change *prepare_expression()* to use a dict of Dataset objects rather than the hdf file, so that prepare_expression can be used in HdfMap without opening the file. In practice, this doesn't work as a complete list of datasets needs to be built, testing each dataset in the list against the hdf file, which is much slower than the old way of only testing the identifiers that exist in the expression. Tests pass except test_compare_many_files, which shows a significant slow down. This commit will be reverted. --- src/hdfmap/eval_functions.py | 57 +++++++++++++++++++++++------------- src/hdfmap/hdfmap_class.py | 38 +++++++++++++++++++++++- tests/test_edge_cases.py | 6 +++- tests/test_eval_functions.py | 6 +++- tests/test_hdfmap_class.py | 28 +++++++++++++++++- 5 files changed, 110 insertions(+), 25 deletions(-) diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index 0d6e53e..7298e77 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -13,6 +13,7 @@ import h5py from types import EllipsisType +from .objects import Group, Dataset from .logging import create_logger # parameters @@ -366,8 +367,9 @@ def select_ids(startswith=''): return data -def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - data_namespace: dict[str, typing.Any] | None, replace_names: dict[str, str]) -> str: +def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], + data_namespace: dict[str, typing.Any] | None, + replace_names: dict[str, str]) -> str: """ Prepare an expression for evaluation using the namespace of the hdf file Returns the modified expression replacing attribute names and alternates with @@ -389,15 +391,12 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict Shorthand variables for expressions can be assigned using replace_names = {'new_name': 'favourite*expression'} - :param hdf_file: h5py.File object :param expression: str expression to be evaluated - :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} - :param data_namespace: dict of {'variable name': value} ** note: values will be added to this dict + :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} + :param data_namespace: dict of {'variable name': value} ** note: attr data will be added to this dict :param replace_names: dict of {'variable_name': expression} :return: str expression """ - #TODO: remove hdf from this function, replace with dict of Dataset objects. - #TODO: Move Dataset objects into seperate file that can be improted # replace names with expressions expression = replace_expression_vars(expression, replace_names) @@ -406,9 +405,9 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict data_namespace = {} # find name@attribute in expression attributes = { - f"attr__{name}_{attr}": dataset_attribute(hdf_file[path], attr) + f"attr__{name}_{attr}": dataset.attrs.get(attr) for name, attr in re_dataset_attributes.findall(expression) # name@attr - if (path := hdf_namespace.get(name, '')) in hdf_file + if (dataset := dataset_namespace.get(name)) } data_namespace.update(attributes) # adds data in the parent function # replace name@attribute in expression @@ -416,7 +415,7 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict # find values with defaults '..?(..)' for match in re_dataset_default.finditer(expression): name, name_default = match.groups() - if name not in hdf_namespace: + if name not in dataset_namespace: expression = expression.replace(match.group(), name_default) else: expression = expression.replace(match.group(), name) @@ -426,15 +425,16 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict names = or_exp.strip('()').split('|') name = next( (n for n in names if n in attributes), - next((n for n in names if hdf_namespace.get(n, '') in hdf_file), names[-1]) + next((n for n in names if n in dataset_namespace), names[-1]) ) expression = expression.replace(or_exp, name) return expression def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - data_namespace: dict[str, typing.Any], replace_names: dict[str, str], - default: typing.Any = DEFAULT, use_stored_data: bool = False): + dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], + replace_names: dict[str, str], default: typing.Any = DEFAULT, + use_stored_data: bool = False): """ Prepare an expression for evaluation using the namespace of the hdf file Returns the modified expression replacing attribute names and alternates with @@ -459,6 +459,7 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names :param hdf_file: h5py.File object :param expression: str expression to be evaluated :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} + :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} ** note: values will be added to this dict :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -466,7 +467,7 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names :return: str expression """ # replace parts of the expression & add attributes to data_namespace - expression = prepare_expression(hdf_file, expression, hdf_namespace, data_namespace, replace_names) + expression = prepare_expression(expression, dataset_namespace, data_namespace, replace_names) # find identifier symbols in expression identifiers = find_identifiers(expression) if use_stored_data: @@ -480,8 +481,9 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - data_namespace: dict[str, typing.Any], replace_names: dict[str, str], - default: typing.Any = DEFAULT, use_stored_data: bool = False, raise_errors: bool = True) -> typing.Any: + dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], + replace_names: dict[str, str], default: typing.Any = DEFAULT, + use_stored_data: bool = False, raise_errors: bool = True) -> typing.Any: """ Evaluate an expression using the namespace of the hdf file @@ -504,6 +506,7 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] :param hdf_file: h5py.File object :param expression: str expression to be evaluated :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} + :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} (note, this object will be updated) :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -518,8 +521,16 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] # if expression is a hdf path, just return the data if expression in hdf_file: return dataset2data(hdf_file[expression]) - expression = prepare_expression_load_data(hdf_file, expression, hdf_namespace, data_namespace, - replace_names, default, use_stored_data) + expression = prepare_expression_load_data( + hdf_file=hdf_file, + expression=expression, + hdf_namespace=hdf_namespace, + dataset_namespace=dataset_namespace, + data_namespace=data_namespace, + replace_names=replace_names, + default=default, + use_stored_data=use_stored_data + ) logger.debug(f"evaluating expression: '{expression}'") # evaluate expression within namespace safe_eval = asteval.Interpreter(user_symbols=data_namespace, use_numpy=True) @@ -531,13 +542,15 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - data_namespace: dict[str, typing.Any], replace_names: dict[str, str], - default: typing.Any = DEFAULT, use_stored_data: bool = False, raise_errors: bool = True) -> str: + dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], + replace_names: dict[str, str], default: typing.Any = DEFAULT, + use_stored_data: bool = False, raise_errors: bool = True) -> str: """ Evaluate a formatted string expression using the namespace of the hdf file :param hdf_file: h5py.File object :param expression: str expression using {name} format specifiers :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} + :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -550,6 +563,7 @@ def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, st hdf_file=hdf_file, expression=expression, hdf_namespace=hdf_namespace, + dataset_namespace=dataset_namespace, data_namespace=data_namespace, replace_names=replace_names, default=default, @@ -586,7 +600,8 @@ def eval(self, expr, lineno=0, show_errors=True, raise_errors=False): hdf_file=hdf, expression=expr, hdf_namespace=self.hdfmap.combined, - data_namespace=self.symtable, + dataset_namespace=self.hdfmap.datasets, + data_namespace=self.symtable or {}, replace_names=self.replace_names, default=self.default_value, use_stored_data=self.use_stored_data diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 4947718..070e1ae 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -11,7 +11,7 @@ from . import load_hdf from .data_holder import disp_dict, DataHolder from .logging import create_logger -from .eval_functions import (expression_safe_name, extra_hdf_data, eval_hdf, HdfMapInterpreter, +from .eval_functions import (extra_hdf_data, eval_hdf, HdfMapInterpreter, prepare_expression_load_data, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) from .objects import Group, Dataset @@ -1072,6 +1072,10 @@ def get_dataholder(self, hdf_file: h5py.File, flatten_scannables: bool = False) scannables['metadata'] = DataHolder(**metadata) return DataHolder(**scannables) + def _get_datasets_in_file(self, hdf_file: h5py.File) -> dict[str, Dataset]: + """get {name: Dataset} dict for Datasets available in File""" + return {name: self.datasets[path] for name, path in self.combined.items() if path in hdf_file} + def eval(self, hdf_file: h5py.File, expression: str, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None, raise_errors: bool = True) -> typing.Any: @@ -1089,6 +1093,7 @@ def eval(self, hdf_file: h5py.File, expression: str, default=DEFAULT, hdf_file=hdf_file, expression=expression, hdf_namespace=self.combined, + dataset_namespace=self._get_datasets_in_file(hdf_file), data_namespace=self._local_data if local_data is None else local_data, replace_names=self.alternate_names, default=default, @@ -1113,6 +1118,7 @@ def format_hdf(self, hdf_file: h5py.File, expression: str, default=DEFAULT, hdf_file=hdf_file, expression=expression, hdf_namespace=self.combined, + dataset_namespace=self._get_datasets_in_file(hdf_file), data_namespace=self._local_data if local_data is None else local_data, replace_names=self.alternate_names, default=default, @@ -1145,6 +1151,36 @@ def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, pr interpreter.use_stored_data = self._use_local_data if prefer_local is None else prefer_local return interpreter + def generate_eval_expression(self, hdf_file: h5py.File, expression: str, default=DEFAULT, + local_data: dict | None = None, prefer_local: bool | None = None) -> tuple[str, dict[str, typing.Any]]: + """ + Evaluate an expression using the namespace of the hdf file, + returning the evaluated expression and dictionary of data for identifiers + + expression, data = HdfMap.generate_eval_expression(hdf, 'signal / (monitor|ic1monitor)') + + This function serves as a useful way to debug expressions for the eval_hdf function. + + :param hdf_file: h5py.File object + :param expression: str expression to be evaluated + :param default: returned if varname not in namespace + :param local_data: dict of additional data to pass to the expression, as {'varname': data} + :param prefer_local: uses values in local_data first if available when True + :return: expression, dict - data namespace + """ + local_data = self._local_data.copy() if local_data is None else local_data.copy() + new_expression = prepare_expression_load_data( + hdf_file=hdf_file, + expression=expression, + hdf_namespace=self.combined, + dataset_namespace=self._get_datasets_in_file(hdf_file), + data_namespace=local_data, + replace_names=self.alternate_names, + default=default, + use_stored_data=self._use_local_data if prefer_local is None else prefer_local, + ) + return new_expression, local_data + def create_dataset_summary(self, hdf_file: h5py.File) -> str: """Create summary of all datasets in file""" return '\n'.join(f"{path:60}: {self.get_string(hdf_file, path)}" for path in self.datasets) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 4e593ca..ca3e471 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -202,16 +202,20 @@ def test_alternate_name_local_data(): f1 = '/dls/science/groups/das/ExampleData/hdfmap_tests/i16/1109527.nxs' f2 = '/dls/science/groups/das/ExampleData/hdfmap_tests/i16/1113658.nxs' m = hdfmap.create_nexus_map(f1) + assert 'cmd' in m and 'scan_command' in m with hdfmap.load_hdf(f1) as nxs: + # f1 contains cmd scan_command1 = m.get_data(nxs, 'scan_command') print(scan_command1) cmd1 = m.eval(nxs, '(cmd|scan_command)') assert scan_command1 != cmd1, 'cmd and scan_command should not be the same' + assert 'cmd' in m._local_data with hdfmap.load_hdf(f2) as nxs: + # f2 doesn't contain cmd, but it is now in the local_data scan_command2 = m.get_data(nxs, 'scan_command') cmd2 = m.eval(nxs, '(cmd|scan_command)') - assert scan_command2 == cmd2, 'cmd and scan_command should be the same' + assert scan_command2 == cmd2, "cmd2 pulling from local but shouldn't" assert cmd1 != cmd2, 'cmd of both files should not be the same' diff --git a/tests/test_eval_functions.py b/tests/test_eval_functions.py index 97a819f..2759cb6 100644 --- a/tests/test_eval_functions.py +++ b/tests/test_eval_functions.py @@ -4,6 +4,7 @@ import os import hdfmap +from hdfmap.objects import Dataset, Group from hdfmap.eval_functions import replace_expression_vars, prepare_expression_load_data @@ -34,9 +35,12 @@ def test_prepare_expression_load_data(): 'x': 'a', } hdf_map = {'idgap': '/entry/instrument/insertion_device/gap'} + grp = Group('insertion_device', '.', {}, [], None, False, None) + ds = Dataset('idgap', [], 31, (31, ), {'units': 'mm'}, grp, None) + dataset_map = {'idgap': ds} data = {} with hdfmap.load_hdf(FILE_HKL) as hdf: - new_expr = prepare_expression_load_data(hdf, expression, hdf_map, data, repl) + new_expr = prepare_expression_load_data(hdf, expression, hdf_map, dataset_map, data, repl) assert new_expr == "attr__idgap_units, a*y" assert data['attr__idgap_units'] == 'mm' diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index b13b8e5..dfb79a5 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -1,9 +1,11 @@ -import h5py + import pytest import sys import os import datetime import hdfmap +from unittest.mock import ANY +from pytest import approx DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'data') FILE_HKL = DATA_FOLDER + "/1049598.nxs" # hkl scan, pilatus @@ -332,3 +334,27 @@ def test_interpreter(hdf_map): ii = hdf_map.create_interpreter() assert abs(ii('abs(mean(max(roi2_sum)))') - 359573) < 0.001, "Expression output gives wrong result" assert 'hkl' in ii('scan_command'), 'Expression output gives wrong result' + + +def test_generate_eval_expression(hdf_map): + + expression = 'signal / (transmission|Transmission) / count_time' + hdf_map.add_named_expression(count_time='int(mean((count_time|counttime|t?(0.5))))') + + with hdfmap.load_hdf(FILE_HKL) as hdf: + new_expression, data = hdf_map.generate_eval_expression( + hdf_file=hdf, + expression=expression, + default=0, + local_data={'Transmission': 0.1}, + prefer_local=False, + ) + print(new_expression) + print(data) + assert new_expression == 'signal / Transmission / int(mean(count_time))' + assert len(data) == 5 + assert data.get('Transmission') == approx(1) + assert data.get('signal') == approx(0) # signal not defined for HdfMap + assert sum(data.get('count_time', 0)) == approx(101) + assert data.get('filename') == '1049598.nxs' + From 259c12988b2cef7218ad6aa3b27fb5d9b7fb532a Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 10:52:41 +0100 Subject: [PATCH 4/9] Revert "prepare expression using Datasets" This reverts commit 410c0c94ea983f9c54dcf2b286e9815468930892. --- src/hdfmap/eval_functions.py | 57 +++++++++++++----------------------- src/hdfmap/hdfmap_class.py | 38 +----------------------- tests/test_edge_cases.py | 6 +--- tests/test_eval_functions.py | 6 +--- tests/test_hdfmap_class.py | 28 +----------------- 5 files changed, 25 insertions(+), 110 deletions(-) diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index 7298e77..0d6e53e 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -13,7 +13,6 @@ import h5py from types import EllipsisType -from .objects import Group, Dataset from .logging import create_logger # parameters @@ -367,9 +366,8 @@ def select_ids(startswith=''): return data -def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], - data_namespace: dict[str, typing.Any] | None, - replace_names: dict[str, str]) -> str: +def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], + data_namespace: dict[str, typing.Any] | None, replace_names: dict[str, str]) -> str: """ Prepare an expression for evaluation using the namespace of the hdf file Returns the modified expression replacing attribute names and alternates with @@ -391,12 +389,15 @@ def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], Shorthand variables for expressions can be assigned using replace_names = {'new_name': 'favourite*expression'} + :param hdf_file: h5py.File object :param expression: str expression to be evaluated - :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} - :param data_namespace: dict of {'variable name': value} ** note: attr data will be added to this dict + :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} + :param data_namespace: dict of {'variable name': value} ** note: values will be added to this dict :param replace_names: dict of {'variable_name': expression} :return: str expression """ + #TODO: remove hdf from this function, replace with dict of Dataset objects. + #TODO: Move Dataset objects into seperate file that can be improted # replace names with expressions expression = replace_expression_vars(expression, replace_names) @@ -405,9 +406,9 @@ def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], data_namespace = {} # find name@attribute in expression attributes = { - f"attr__{name}_{attr}": dataset.attrs.get(attr) + f"attr__{name}_{attr}": dataset_attribute(hdf_file[path], attr) for name, attr in re_dataset_attributes.findall(expression) # name@attr - if (dataset := dataset_namespace.get(name)) + if (path := hdf_namespace.get(name, '')) in hdf_file } data_namespace.update(attributes) # adds data in the parent function # replace name@attribute in expression @@ -415,7 +416,7 @@ def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], # find values with defaults '..?(..)' for match in re_dataset_default.finditer(expression): name, name_default = match.groups() - if name not in dataset_namespace: + if name not in hdf_namespace: expression = expression.replace(match.group(), name_default) else: expression = expression.replace(match.group(), name) @@ -425,16 +426,15 @@ def prepare_expression(expression: str, dataset_namespace: dict[str, Dataset], names = or_exp.strip('()').split('|') name = next( (n for n in names if n in attributes), - next((n for n in names if n in dataset_namespace), names[-1]) + next((n for n in names if hdf_namespace.get(n, '') in hdf_file), names[-1]) ) expression = expression.replace(or_exp, name) return expression def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], - replace_names: dict[str, str], default: typing.Any = DEFAULT, - use_stored_data: bool = False): + data_namespace: dict[str, typing.Any], replace_names: dict[str, str], + default: typing.Any = DEFAULT, use_stored_data: bool = False): """ Prepare an expression for evaluation using the namespace of the hdf file Returns the modified expression replacing attribute names and alternates with @@ -459,7 +459,6 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names :param hdf_file: h5py.File object :param expression: str expression to be evaluated :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} - :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} ** note: values will be added to this dict :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -467,7 +466,7 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names :return: str expression """ # replace parts of the expression & add attributes to data_namespace - expression = prepare_expression(expression, dataset_namespace, data_namespace, replace_names) + expression = prepare_expression(hdf_file, expression, hdf_namespace, data_namespace, replace_names) # find identifier symbols in expression identifiers = find_identifiers(expression) if use_stored_data: @@ -481,9 +480,8 @@ def prepare_expression_load_data(hdf_file: h5py.File, expression: str, hdf_names def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], - replace_names: dict[str, str], default: typing.Any = DEFAULT, - use_stored_data: bool = False, raise_errors: bool = True) -> typing.Any: + data_namespace: dict[str, typing.Any], replace_names: dict[str, str], + default: typing.Any = DEFAULT, use_stored_data: bool = False, raise_errors: bool = True) -> typing.Any: """ Evaluate an expression using the namespace of the hdf file @@ -506,7 +504,6 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] :param hdf_file: h5py.File object :param expression: str expression to be evaluated :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} - :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} (note, this object will be updated) :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -521,16 +518,8 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] # if expression is a hdf path, just return the data if expression in hdf_file: return dataset2data(hdf_file[expression]) - expression = prepare_expression_load_data( - hdf_file=hdf_file, - expression=expression, - hdf_namespace=hdf_namespace, - dataset_namespace=dataset_namespace, - data_namespace=data_namespace, - replace_names=replace_names, - default=default, - use_stored_data=use_stored_data - ) + expression = prepare_expression_load_data(hdf_file, expression, hdf_namespace, data_namespace, + replace_names, default, use_stored_data) logger.debug(f"evaluating expression: '{expression}'") # evaluate expression within namespace safe_eval = asteval.Interpreter(user_symbols=data_namespace, use_numpy=True) @@ -542,15 +531,13 @@ def eval_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str] def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, str], - dataset_namespace: dict[str, Dataset], data_namespace: dict[str, typing.Any], - replace_names: dict[str, str], default: typing.Any = DEFAULT, - use_stored_data: bool = False, raise_errors: bool = True) -> str: + data_namespace: dict[str, typing.Any], replace_names: dict[str, str], + default: typing.Any = DEFAULT, use_stored_data: bool = False, raise_errors: bool = True) -> str: """ Evaluate a formatted string expression using the namespace of the hdf file :param hdf_file: h5py.File object :param expression: str expression using {name} format specifiers :param hdf_namespace: dict of {'variable name': '/hdf/dataset/path'} - :param dataset_namespace: dict of {'variable name': Dataset object from HdfMap} :param data_namespace: dict of {'variable name': value} :param replace_names: dict of {'variable_name': expression} :param default: returned if varname not in namespace @@ -563,7 +550,6 @@ def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, st hdf_file=hdf_file, expression=expression, hdf_namespace=hdf_namespace, - dataset_namespace=dataset_namespace, data_namespace=data_namespace, replace_names=replace_names, default=default, @@ -600,8 +586,7 @@ def eval(self, expr, lineno=0, show_errors=True, raise_errors=False): hdf_file=hdf, expression=expr, hdf_namespace=self.hdfmap.combined, - dataset_namespace=self.hdfmap.datasets, - data_namespace=self.symtable or {}, + data_namespace=self.symtable, replace_names=self.replace_names, default=self.default_value, use_stored_data=self.use_stored_data diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 070e1ae..4947718 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -11,7 +11,7 @@ from . import load_hdf from .data_holder import disp_dict, DataHolder from .logging import create_logger -from .eval_functions import (extra_hdf_data, eval_hdf, HdfMapInterpreter, prepare_expression_load_data, +from .eval_functions import (expression_safe_name, extra_hdf_data, eval_hdf, HdfMapInterpreter, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) from .objects import Group, Dataset @@ -1072,10 +1072,6 @@ def get_dataholder(self, hdf_file: h5py.File, flatten_scannables: bool = False) scannables['metadata'] = DataHolder(**metadata) return DataHolder(**scannables) - def _get_datasets_in_file(self, hdf_file: h5py.File) -> dict[str, Dataset]: - """get {name: Dataset} dict for Datasets available in File""" - return {name: self.datasets[path] for name, path in self.combined.items() if path in hdf_file} - def eval(self, hdf_file: h5py.File, expression: str, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None, raise_errors: bool = True) -> typing.Any: @@ -1093,7 +1089,6 @@ def eval(self, hdf_file: h5py.File, expression: str, default=DEFAULT, hdf_file=hdf_file, expression=expression, hdf_namespace=self.combined, - dataset_namespace=self._get_datasets_in_file(hdf_file), data_namespace=self._local_data if local_data is None else local_data, replace_names=self.alternate_names, default=default, @@ -1118,7 +1113,6 @@ def format_hdf(self, hdf_file: h5py.File, expression: str, default=DEFAULT, hdf_file=hdf_file, expression=expression, hdf_namespace=self.combined, - dataset_namespace=self._get_datasets_in_file(hdf_file), data_namespace=self._local_data if local_data is None else local_data, replace_names=self.alternate_names, default=default, @@ -1151,36 +1145,6 @@ def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, pr interpreter.use_stored_data = self._use_local_data if prefer_local is None else prefer_local return interpreter - def generate_eval_expression(self, hdf_file: h5py.File, expression: str, default=DEFAULT, - local_data: dict | None = None, prefer_local: bool | None = None) -> tuple[str, dict[str, typing.Any]]: - """ - Evaluate an expression using the namespace of the hdf file, - returning the evaluated expression and dictionary of data for identifiers - - expression, data = HdfMap.generate_eval_expression(hdf, 'signal / (monitor|ic1monitor)') - - This function serves as a useful way to debug expressions for the eval_hdf function. - - :param hdf_file: h5py.File object - :param expression: str expression to be evaluated - :param default: returned if varname not in namespace - :param local_data: dict of additional data to pass to the expression, as {'varname': data} - :param prefer_local: uses values in local_data first if available when True - :return: expression, dict - data namespace - """ - local_data = self._local_data.copy() if local_data is None else local_data.copy() - new_expression = prepare_expression_load_data( - hdf_file=hdf_file, - expression=expression, - hdf_namespace=self.combined, - dataset_namespace=self._get_datasets_in_file(hdf_file), - data_namespace=local_data, - replace_names=self.alternate_names, - default=default, - use_stored_data=self._use_local_data if prefer_local is None else prefer_local, - ) - return new_expression, local_data - def create_dataset_summary(self, hdf_file: h5py.File) -> str: """Create summary of all datasets in file""" return '\n'.join(f"{path:60}: {self.get_string(hdf_file, path)}" for path in self.datasets) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index ca3e471..4e593ca 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -202,20 +202,16 @@ def test_alternate_name_local_data(): f1 = '/dls/science/groups/das/ExampleData/hdfmap_tests/i16/1109527.nxs' f2 = '/dls/science/groups/das/ExampleData/hdfmap_tests/i16/1113658.nxs' m = hdfmap.create_nexus_map(f1) - assert 'cmd' in m and 'scan_command' in m with hdfmap.load_hdf(f1) as nxs: - # f1 contains cmd scan_command1 = m.get_data(nxs, 'scan_command') print(scan_command1) cmd1 = m.eval(nxs, '(cmd|scan_command)') assert scan_command1 != cmd1, 'cmd and scan_command should not be the same' - assert 'cmd' in m._local_data with hdfmap.load_hdf(f2) as nxs: - # f2 doesn't contain cmd, but it is now in the local_data scan_command2 = m.get_data(nxs, 'scan_command') cmd2 = m.eval(nxs, '(cmd|scan_command)') - assert scan_command2 == cmd2, "cmd2 pulling from local but shouldn't" + assert scan_command2 == cmd2, 'cmd and scan_command should be the same' assert cmd1 != cmd2, 'cmd of both files should not be the same' diff --git a/tests/test_eval_functions.py b/tests/test_eval_functions.py index 2759cb6..97a819f 100644 --- a/tests/test_eval_functions.py +++ b/tests/test_eval_functions.py @@ -4,7 +4,6 @@ import os import hdfmap -from hdfmap.objects import Dataset, Group from hdfmap.eval_functions import replace_expression_vars, prepare_expression_load_data @@ -35,12 +34,9 @@ def test_prepare_expression_load_data(): 'x': 'a', } hdf_map = {'idgap': '/entry/instrument/insertion_device/gap'} - grp = Group('insertion_device', '.', {}, [], None, False, None) - ds = Dataset('idgap', [], 31, (31, ), {'units': 'mm'}, grp, None) - dataset_map = {'idgap': ds} data = {} with hdfmap.load_hdf(FILE_HKL) as hdf: - new_expr = prepare_expression_load_data(hdf, expression, hdf_map, dataset_map, data, repl) + new_expr = prepare_expression_load_data(hdf, expression, hdf_map, data, repl) assert new_expr == "attr__idgap_units, a*y" assert data['attr__idgap_units'] == 'mm' diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index dfb79a5..b13b8e5 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -1,11 +1,9 @@ - +import h5py import pytest import sys import os import datetime import hdfmap -from unittest.mock import ANY -from pytest import approx DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'data') FILE_HKL = DATA_FOLDER + "/1049598.nxs" # hkl scan, pilatus @@ -334,27 +332,3 @@ def test_interpreter(hdf_map): ii = hdf_map.create_interpreter() assert abs(ii('abs(mean(max(roi2_sum)))') - 359573) < 0.001, "Expression output gives wrong result" assert 'hkl' in ii('scan_command'), 'Expression output gives wrong result' - - -def test_generate_eval_expression(hdf_map): - - expression = 'signal / (transmission|Transmission) / count_time' - hdf_map.add_named_expression(count_time='int(mean((count_time|counttime|t?(0.5))))') - - with hdfmap.load_hdf(FILE_HKL) as hdf: - new_expression, data = hdf_map.generate_eval_expression( - hdf_file=hdf, - expression=expression, - default=0, - local_data={'Transmission': 0.1}, - prefer_local=False, - ) - print(new_expression) - print(data) - assert new_expression == 'signal / Transmission / int(mean(count_time))' - assert len(data) == 5 - assert data.get('Transmission') == approx(1) - assert data.get('signal') == approx(0) # signal not defined for HdfMap - assert sum(data.get('count_time', 0)) == approx(101) - assert data.get('filename') == '1049598.nxs' - From adf227afe37efd85aca55da9af8450f224659436 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 11:21:40 +0100 Subject: [PATCH 5/9] add generate_eval_expression This new function is useful for debugging and returns the prepared expression and dict of data, prior to evaluation. ## hdfmap_class.py - add *HdfMap.generate_eval_expression()* - add test for this ## reloader_class.py - add *HdfLoader.generate_expression()* All tests pass. --- src/hdfmap/hdfmap_class.py | 34 ++++++++++++++++++++++++++++- src/hdfmap/reloader_class.py | 42 +++++++++++++++++++++++++++++++++--- tests/test_hdfmap_class.py | 24 +++++++++++++++++++++ 3 files changed, 96 insertions(+), 4 deletions(-) diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 4947718..524498e 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -11,7 +11,7 @@ from . import load_hdf from .data_holder import disp_dict, DataHolder from .logging import create_logger -from .eval_functions import (expression_safe_name, extra_hdf_data, eval_hdf, HdfMapInterpreter, +from .eval_functions import (extra_hdf_data, eval_hdf, HdfMapInterpreter, prepare_expression_load_data, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) from .objects import Group, Dataset @@ -1145,6 +1145,38 @@ def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, pr interpreter.use_stored_data = self._use_local_data if prefer_local is None else prefer_local return interpreter + def generate_eval_expression(self, hdf_file: h5py.File, expression: str, default=DEFAULT, + local_data: dict | None = None, prefer_local: bool | None = None) -> tuple[str, dict[str, typing.Any]]: + """ + Evaluate an expression using the namespace of the hdf file, + returning the evaluated expression and dictionary of data for identifiers + + expression, data = HdfMap.generate_eval_expression(hdf, 'signal / (monitor|ic1monitor)') + + This function serves as a useful way to debug expressions for the eval_hdf function. + Note that the hdf file object must be included as the way the expression is evaluated + means individual expression components are checked against the HdfMap namespace and + the hdf file, allowing lazy loading of data (loading only the data needed). + + :param hdf_file: h5py.File object + :param expression: str expression to be evaluated + :param default: returned if varname not in namespace + :param local_data: dict of additional data to pass to the expression, as {'varname': data} + :param prefer_local: uses values in local_data first if available when True + :return: expression, dict - data namespace + """ + local_data = self._local_data.copy() if local_data is None else local_data.copy() + new_expression = prepare_expression_load_data( + hdf_file=hdf_file, + expression=expression, + hdf_namespace=self.combined, + data_namespace=local_data, + replace_names=self.alternate_names, + default=default, + use_stored_data=self._use_local_data if prefer_local is None else prefer_local, + ) + return new_expression, local_data + def create_dataset_summary(self, hdf_file: h5py.File) -> str: """Create summary of all datasets in file""" return '\n'.join(f"{path:60}: {self.get_string(hdf_file, path)}" for path in self.datasets) diff --git a/src/hdfmap/reloader_class.py b/src/hdfmap/reloader_class.py index 01cfcdd..9cf15f7 100644 --- a/src/hdfmap/reloader_class.py +++ b/src/hdfmap/reloader_class.py @@ -1,16 +1,22 @@ """ Reloader class +The ReLoader class is an object that contains a HdfMap and a filename, +allowing the file to be opened only when data is requested. """ import os import h5py import numpy as np +from typing import Any from . import load_hdf, HdfMap, NexusMap from .file_functions import create_hdf_map, create_nexus_map from .eval_functions import DEFAULT +Index = int | tuple | slice + + class HdfLoader: """ HDF Loader contains the filename and hdfmap for a HDF file, the hdfmap contains all the dataset paths and a @@ -96,7 +102,7 @@ def find_names(self, string: str) -> list[str]: """ return self.map.find_names(string) - def get_data(self, *name_or_path, index: slice = (), default=None, direct_load=False): + def get_data(self, *name_or_path, index: Index = (), default=None, direct_load=False): """ Return data from dataset in file, converted into either datetime, str or squeezed numpy.array objects See hdfmap.eval_functions.dataset2data for more information. @@ -112,7 +118,7 @@ def get_data(self, *name_or_path, index: slice = (), default=None, direct_load=F return out[0] return out - def get_string(self, *name_or_path, index: slice = (), default='', units=False): + def get_string(self, *name_or_path, index: Index = (), default='', units=False): """ Return data from dataset in file, converted into summary string See hdfmap.eval_functions.dataset2data for more information. @@ -128,7 +134,7 @@ def get_string(self, *name_or_path, index: slice = (), default='', units=False): return out[0] return out - def get_image(self, index: slice = None) -> np.ndarray: + def get_image(self, index: Index | None = None) -> np.ndarray | None: """ Get image data from file, using default image path :param index: (slice,) or None to take the middle image @@ -151,6 +157,36 @@ def summary(self) -> str: with self._load() as hdf: return self.map.create_dataset_summary(hdf) + def generate_expression(self, expression: str, default=DEFAULT, + prefer_local: bool | None = None) -> tuple[str, dict[str, Any]]: + """ + Evaluate an expression using the namespace of the hdf file, + returning the evaluated expression and dictionary of data for identifiers + + expression, data = self.generate_eval_expression('signal / (monitor|ic1monitor)') + + This function serves as a useful way to debug expressions for the eval_hdf function. + Note that the hdf file object must be included as the way the expression is evaluated + means individual expression components are checked against the HdfMap namespace and + the hdf file, allowing lazy loading of data (loading only the data needed). + + :param expression: str expression to be evaluated + :param default: returned if varname not in namespace + :param prefer_local: uses values in local_data first if available when True + :return: expression, dict - data namespace + """ + prefer_local = self._prefer_local_data if prefer_local is None else prefer_local + if prefer_local and expression in self._local_data: + return expression, {expression: self._local_data[expression]} + with self._load() as hdf: + return self.map.generate_eval_expression( + hdf_file=hdf, + expression=expression, + default=default, + local_data=self._local_data, + prefer_local=prefer_local, + ) + def eval(self, expression: str, default=DEFAULT, prefer_local: bool | None = None, raise_errors: bool = True): """ Evaluate an expression using the namespace of the hdf file diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index b13b8e5..8745c07 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -332,3 +332,27 @@ def test_interpreter(hdf_map): ii = hdf_map.create_interpreter() assert abs(ii('abs(mean(max(roi2_sum)))') - 359573) < 0.001, "Expression output gives wrong result" assert 'hkl' in ii('scan_command'), 'Expression output gives wrong result' + + +def test_generate_eval_expression(hdf_map): + + expression = 'signal / (transmission|Transmission) / count_time' + hdf_map.add_named_expression(count_time='int(mean((count_time|counttime|t?(0.5))))') + + with hdfmap.load_hdf(FILE_HKL) as hdf: + new_expression, data = hdf_map.generate_eval_expression( + hdf_file=hdf, + expression=expression, + default=0, + local_data={'Transmission': 0.1}, + prefer_local=False, + ) + print(new_expression) + print(data) + assert new_expression == 'signal / Transmission / int(mean(count_time))' + assert len(data) == 5 + assert data.get('Transmission') == pytest.approx(1) + assert data.get('signal') == pytest.approx(0) # signal not defined for HdfMap + assert sum(data.get('count_time', 0)) == pytest.approx(101) + assert data.get('filename') == '1049598.nxs' + From a03e503eca014e6f052653b73ba931865619d724 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 11:24:57 +0100 Subject: [PATCH 6/9] improve docs --- src/hdfmap/reloader_class.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hdfmap/reloader_class.py b/src/hdfmap/reloader_class.py index 9cf15f7..6e30e6f 100644 --- a/src/hdfmap/reloader_class.py +++ b/src/hdfmap/reloader_class.py @@ -23,12 +23,14 @@ class HdfLoader: namespace, allowing data to be called from the file using variable names, loading only the required datasets for each operation. - ### E.G. hdf = HdfLoader('file.hdf') [data1, data2] = hdf.get_data(*['dataset_name_1', 'dataset_name_2']) data = hdf.eval('dataset_name_1 * 100 + 2') string = hdf.format('my data is {dataset_name_1:.2f}') print(hdf.summary()) + + :param hdf_filename: path to HDF file + :param hdf_map: HdfMap instance """ def __init__(self, hdf_filename: str, hdf_map: HdfMap | NexusMap | None = None): @@ -255,11 +257,14 @@ class NexusLoader(HdfLoader): contains the filename and hdfmap for a NeXus file, the hdfmap contains all the dataset paths and a namespace, allowing data to be called from the file using variable names, loading only the required datasets for each operation. - E.G. + hdf = NexusLoader('file.hdf') [data1, data2] = hdf.get_data(['dataset_name_1', 'dataset_name_2']) data = hdf.eval('dataset_name_1 * 100 + 2') string = hdf.format('my data is {dataset_name_1:.2f}') + + :param nxs_filename: path to HDF file + :param hdf_map: NexusMap instance, or None to generate """ map: NexusMap From dcf818048c5fef4ae082cd8994eeb504058872b2 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 11:56:11 +0100 Subject: [PATCH 7/9] refactor Interpreter Move HdfMap Interpreter from HdfMap class to reloader class. All tests pass. --- src/hdfmap/eval_functions.py | 34 ------------------------- src/hdfmap/hdfmap_class.py | 27 +------------------- src/hdfmap/reloader_class.py | 48 +++++++++++++++++++++++++++++++++--- tests/test_hdfmap_class.py | 6 ----- tests/test_reloader_class.py | 9 ++++++- 5 files changed, 54 insertions(+), 70 deletions(-) diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index 0d6e53e..2a61875 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -558,37 +558,3 @@ def format_hdf(hdf_file: h5py.File, expression: str, hdf_namespace: dict[str, st ) -class HdfMapInterpreter(asteval.Interpreter): - """ - HdfMap implementation of asteval.Interpreter - - Expression is parsed for patterns and loads HDF data before evaluation - - m = HdfMap('file.nxs') - ii = HdfMapInterpreter(m, replace_names={}, default='', **kwargs) - out = ii.eval('expression') - - :param hdfmap: HdfMap instance (including hdfmap.filename pointing to the HDF file) - :param replace_names: dict of {'variable_name': expression} - :param default: returned if varname not in namespace - :param kwargs: keyword arguments passed to asteval.Interpreter - """ - def __init__(self, hdfmap, replace_names: dict[str, str], default: typing.Any = DEFAULT, **kws): - super().__init__(**kws) - self.hdfmap = hdfmap - self.replace_names = replace_names - self.default_value = default - self.use_stored_data = False - - def eval(self, expr, lineno=0, show_errors=True, raise_errors=False): - with self.hdfmap.load_hdf() as hdf: - new_expression = prepare_expression_load_data( - hdf_file=hdf, - expression=expr, - hdf_namespace=self.hdfmap.combined, - data_namespace=self.symtable, - replace_names=self.replace_names, - default=self.default_value, - use_stored_data=self.use_stored_data - ) - return super().eval(new_expression, lineno, show_errors, raise_errors) diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 524498e..9ae002c 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -11,7 +11,7 @@ from . import load_hdf from .data_holder import disp_dict, DataHolder from .logging import create_logger -from .eval_functions import (extra_hdf_data, eval_hdf, HdfMapInterpreter, prepare_expression_load_data, +from .eval_functions import (extra_hdf_data, eval_hdf, prepare_expression_load_data, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) from .objects import Group, Dataset @@ -1120,31 +1120,6 @@ def format_hdf(self, hdf_file: h5py.File, expression: str, default=DEFAULT, raise_errors=raise_errors ) - def create_interpreter(self, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None) -> HdfMapInterpreter: - """ - Create an interpreter object for the current file - The interpreter is a sub-class of asteval.Interpreter that parses expressions for hdfmap eval patters - and loads data when required. - - The hdf file self.filename is used to extract data and is only opened during evaluation. - - ii = HdfMap.create_interpreter() - out = ii.eval('expression') - - :param default: returned if varname not in namespace - :param local_data: replaces the HdfMap local_data attribute for this file - :param prefer_local: uses values in local_data first if available when True - """ - interpreter = HdfMapInterpreter( - hdfmap=self, - replace_names=self.alternate_names, - default=default, - user_symbols=self._local_data if local_data is None else local_data, - use_numpy=True - ) - interpreter.use_stored_data = self._use_local_data if prefer_local is None else prefer_local - return interpreter - def generate_eval_expression(self, hdf_file: h5py.File, expression: str, default=DEFAULT, local_data: dict | None = None, prefer_local: bool | None = None) -> tuple[str, dict[str, typing.Any]]: """ diff --git a/src/hdfmap/reloader_class.py b/src/hdfmap/reloader_class.py index 6e30e6f..9ac9448 100644 --- a/src/hdfmap/reloader_class.py +++ b/src/hdfmap/reloader_class.py @@ -5,14 +5,15 @@ """ import os +from typing import Any + import h5py import numpy as np -from typing import Any +from asteval import Interpreter from . import load_hdf, HdfMap, NexusMap from .file_functions import create_hdf_map, create_nexus_map -from .eval_functions import DEFAULT - +from .eval_functions import DEFAULT, prepare_expression_load_data Index = int | tuple | slice @@ -277,3 +278,44 @@ def get_plot_data(self) -> dict: """Return dict of useful plot data""" with self._load() as hdf: return self.map.get_plot_data(hdf) + + +class HdfMapInterpreter(Interpreter): + """ + HdfMap implementation of asteval.Interpreter + + Expression is parsed for patterns and loads HDF data before evaluation. + + ii = HdfMapInterpreter('file.nxs', replace_names={}, default='', **kwargs) + out = ii.eval('expression') + + :param filename: path to HDF file + :param hdfmap: HdfMap instance + :param replace_names: dict of {'variable_name': expression} + :param default: returned if varname not in namespace + :param kwargs: keyword arguments passed to asteval.Interpreter + """ + def __init__(self, filename: str, hdfmap: HdfMap | NexusMap | None = None, + replace_names: dict[str, str] | None = None, + default: Any = DEFAULT, **kws): + super().__init__(**kws) + self.filename = filename + if hdfmap is None: + hdfmap = create_nexus_map(filename) if filename.endswith('.nxs') else create_hdf_map(filename) + self.hdfmap = hdfmap + self.replace_names: dict[str, str] = replace_names or {} + self.default_value = default + self.use_stored_data = False + + def eval(self, expr, lineno=0, show_errors=True, raise_errors=False): + with load_hdf(self.filename) as hdf: + new_expression = prepare_expression_load_data( + hdf_file=hdf, + expression=expr, + hdf_namespace=self.hdfmap.combined, + data_namespace=self.symtable or {}, + replace_names=self.replace_names, + default=self.default_value, + use_stored_data=self.use_stored_data + ) + return super().eval(new_expression, lineno, show_errors, raise_errors) diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index 8745c07..3e7da16 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -328,12 +328,6 @@ def test_roi(hdf_map): assert abs(out - 195839) < 0.001, "ROI rmbkg gives wrong result" -def test_interpreter(hdf_map): - ii = hdf_map.create_interpreter() - assert abs(ii('abs(mean(max(roi2_sum)))') - 359573) < 0.001, "Expression output gives wrong result" - assert 'hkl' in ii('scan_command'), 'Expression output gives wrong result' - - def test_generate_eval_expression(hdf_map): expression = 'signal / (transmission|Transmission) / count_time' diff --git a/tests/test_reloader_class.py b/tests/test_reloader_class.py index 8a3e8b7..4354cd1 100644 --- a/tests/test_reloader_class.py +++ b/tests/test_reloader_class.py @@ -1,7 +1,9 @@ import os -from hdfmap import HdfLoader, NexusLoader +from hdfmap import HdfLoader, NexusLoader, NexusMap +from hdfmap.reloader_class import HdfMapInterpreter DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'data') +FILE_HKL = DATA_FOLDER + "/1049598.nxs" # hkl scan, pilatus FILE_NEW_NEXUS = DATA_FOLDER + '/1040323.nxs' # new nexus format FILE_3D_NEXUS = DATA_FOLDER + '/i06-353130.nxs' # new nexus format @@ -40,3 +42,8 @@ def test_nexus_reloader(): 'signal_data', 'axes_labels', 'signal_labels', 'data', 'title'} assert data.keys() >= KEYS + +def test_interpreter(): + ii = HdfMapInterpreter(filename=FILE_HKL) + assert abs(ii('abs(mean(max(roi2_sum)))') - 359573) < 0.001, "Expression output gives wrong result" + assert 'hkl' in ii('scan_command'), 'Expression output gives wrong result' From dac3116be56b301008d10e483ac7e3aa864cfbf4 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 12:03:41 +0100 Subject: [PATCH 8/9] docs for generate_ids --- src/hdfmap/hdfmap_class.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 9ae002c..09e256f 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -815,15 +815,19 @@ def generate_ids(self, *names: str, modify_missing: bool = True) -> list[str]: Will return the path identifier of the given name if the name is in the namespace, otherwise a valid identifier will be generated. - xlabel, ylabel = generate_axis_labels('axes', 'signal') - generate_axis_labels('my/data/label', modify_missing=True) #-> ['label', ] - generate_axis_labels('(x-y)/y', modify_missing=False) #-> ['(x-y)/y', ] + xlabel, ylabel = generate_ids('axes', 'signal') + generate_ids('my/data/label', modify_missing=True) #-> ['label', ] + generate_ids('(x-y)/y', modify_missing=False) #-> ['(x-y)/y', ] + + See also: get_default_name + + Both generate_ids and get_default_name return names from paths or alternate names, + however generate_ids just takes the last element of the path. :param names: names to generate axis labels for :param modify_missing: if True, modifies names even if they are not in namespace :return: list of axis labels as valid identifiers """ - #TODO: use get_default_names or similar here, or leave it as different return [ generate_identifier(self.combined.get(name, name)) if modify_missing else ( generate_identifier(self.combined[name]) if name in self.combined else name From cdd596bd2401da1cd4bffa71b3ebb6b291600ed7 Mon Sep 17 00:00:00 2001 From: Dan Porter Date: Fri, 28 Aug 2026 14:41:29 +0100 Subject: [PATCH 9/9] add merge_default_names ## eval_functions.py - refactor *prepare_expression*, adding seperate functions for each replacement. ## hdfmap_class.py - add *HdfMap.merge_default_names* - add tests --- src/hdfmap/eval_functions.py | 118 ++++++++++++++++++++++++++--------- src/hdfmap/hdfmap_class.py | 19 +++++- src/hdfmap/reloader_class.py | 2 +- tests/test_hdfmap_class.py | 8 +++ tests/test_nexus.py | 4 ++ 5 files changed, 120 insertions(+), 31 deletions(-) diff --git a/src/hdfmap/eval_functions.py b/src/hdfmap/eval_functions.py index 2a61875..3e44f5f 100644 --- a/src/hdfmap/eval_functions.py +++ b/src/hdfmap/eval_functions.py @@ -13,6 +13,7 @@ import h5py from types import EllipsisType +from .objects import Dataset from .logging import create_logger # parameters @@ -234,7 +235,7 @@ def dataset2str(dataset: h5py.Dataset, index: int | slice | tuple[slice, ...] = return str(np.squeeze(dataset[index])) # other np.ndarray -def dataset_attribute(dataset: h5py.Dataset, attribute: str) -> str: +def dataset_attribute(dataset: h5py.Dataset | Dataset, attribute: str) -> str: """ Return attribute of dataset """ @@ -277,6 +278,36 @@ def find_or_expressions(expr: str) -> set[str]: return set(alternates) +def replace_or_expressions(expr: str, path_map: dict[str, str], path_check: h5py.File | list | dict, + data: dict[str, typing.Any]) -> str: + """ + Replaces instances of (a|b|c)-OR-expressions within the string. + The first item of the OR-expression that is found in data or if the + associated path is found in path_check will replace the expression. + + path_map = {'x': '/entry/data/x', 'y': '/entry/data/y'} + path_check = h5py.File('mydata.h5') # file contains x path but not y path + expression = '(y|x)' + new = replace_or_expressions(expression, path_map, path_check, {}) + # new == 'x' + + :param expr: string expression with components to replace + :param path_map: dict mapping from variable name to path as found in path_check + :param path_check: Hdf file to check path in, or could be a list or dict of paths. + :param data: dict mapping of variable name to data + :return: expression with replaced components + """ + while or_expressions := find_or_expressions(expr): + for or_exp in or_expressions: + names = or_exp.strip('()').split('|') + name = next( + (n for n in names if n in data), + next((n for n in names if path_map.get(n, '') in path_check), names[-1]) + ) + expr = expr.replace(or_exp, name) + return expr + + def replace_expression_vars(expr: str, mapping: dict[str, str]) -> str: """ Replace variable names in an expression @@ -304,6 +335,57 @@ def replace_expression_vars(expr: str, mapping: dict[str, str]) -> str: return expr +def replace_defaults(expr: str, path_map: dict[str, str]) -> str: + """ + Replace default specifiers in an expression + A default specifier is given as 'a?(b)', where a will be + returned if 'a' is in path_map, otherwise 'b' is returned. + + path_map = {'x': '/entry/data/x', 'y': '/entry/data/y'} + expression = 'x + y?(10) + z?(20)' + result = replace_defaults(expression, path_map) + # result == 'x + y + 20' + + :param expr: string expression with components to replace + :param path_map: dict mapping from variable name to path + :return: expression with replaced components + """ + for match in re_dataset_default.finditer(expr): + name, name_default = match.groups() + if name not in path_map: + expr = expr.replace(match.group(), name_default) + else: + expr = expr.replace(match.group(), name) + return expr + + +def replace_attributes(expr: str, path_map: dict[str, str], + hdf_file: h5py.File | dict[str, Dataset]) -> tuple[str, dict]: + """ + Find sub-strings of the form 'name@attribute', and replace this with + the attribute associated with the dataset specified by name. + + path_map = {'x': '/entry/data/x', 'y': '/entry/data/y'} + hdf_file = h5py.File('mydata.h5'), where dataset('/entry/data/x') as attribute 'units'='mm' + expression = 'x x@units' + result = replace_attributes(expression, path_map, hdf_file) + # result == 'x mm' + + :param expr: string expression with components to replace + :param path_map: dict mapping from variable name to path + :param hdf_file: h5py.File object + :return: expression with replaced components + """ + attributes = { + f"attr__{name}_{attr}": dataset_attribute(hdf_file[path], attr) + for name, attr in re_dataset_attributes.findall(expr) # name@attr + if (path := path_map.get(name, '')) in hdf_file + } + # replace name@attribute in expression + expr = re_dataset_attributes.sub(r'attr__\g<1>_\g<2>', expr) + return expr, attributes + + def extra_hdf_data(hdf_file: h5py.File) -> dict: """Extract filename, filepath and other additional data fom hdf file""" filepath = getattr(hdf_file, 'filename', 'unknown') @@ -396,39 +478,17 @@ def prepare_expression(hdf_file: h5py.File, expression: str, hdf_namespace: dict :param replace_names: dict of {'variable_name': expression} :return: str expression """ - #TODO: remove hdf from this function, replace with dict of Dataset objects. - #TODO: Move Dataset objects into seperate file that can be improted - + if data_namespace is None: + data_namespace = {} # note the link to the parent namespace must be preserved # replace names with expressions expression = replace_expression_vars(expression, replace_names) - - if data_namespace is None: - data_namespace = {} - # find name@attribute in expression - attributes = { - f"attr__{name}_{attr}": dataset_attribute(hdf_file[path], attr) - for name, attr in re_dataset_attributes.findall(expression) # name@attr - if (path := hdf_namespace.get(name, '')) in hdf_file - } + # replace name@attribute in expression and add values to data_namespace + expression, attributes = replace_attributes(expression, hdf_namespace, hdf_file) data_namespace.update(attributes) # adds data in the parent function - # replace name@attribute in expression - expression = re_dataset_attributes.sub(r'attr__\g<1>_\g<2>', expression) # find values with defaults '..?(..)' - for match in re_dataset_default.finditer(expression): - name, name_default = match.groups() - if name not in hdf_namespace: - expression = expression.replace(match.group(), name_default) - else: - expression = expression.replace(match.group(), name) + expression = replace_defaults(expression, hdf_namespace) # find alternate names '(opt1|opt2|opt3)' - while or_expressions := find_or_expressions(expression): - for or_exp in or_expressions: - names = or_exp.strip('()').split('|') - name = next( - (n for n in names if n in attributes), - next((n for n in names if hdf_namespace.get(n, '') in hdf_file), names[-1]) - ) - expression = expression.replace(or_exp, name) + expression = replace_or_expressions(expression, hdf_namespace, hdf_file, attributes) return expression diff --git a/src/hdfmap/hdfmap_class.py b/src/hdfmap/hdfmap_class.py index 09e256f..4d7f513 100644 --- a/src/hdfmap/hdfmap_class.py +++ b/src/hdfmap/hdfmap_class.py @@ -3,6 +3,7 @@ """ import json import typing +import re from collections import defaultdict import numpy as np @@ -13,7 +14,9 @@ from .logging import create_logger from .eval_functions import (extra_hdf_data, eval_hdf, prepare_expression_load_data, format_hdf, dataset2data, dataset2str, is_image, attrs2dict, - DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name) + DEFAULT, SEP, generate_identifier, build_hdf_path, generate_alt_name, + replace_or_expressions, replace_expression_vars, replace_defaults, + find_identifiers) from .objects import Group, Dataset # parameters @@ -599,6 +602,20 @@ def get_default_name(self, name_or_path: str) -> str | None: return self.groups[path].name return None + def merge_default_names(self, expression: str) -> str: + """Replace names in the expression with dataset default names""" + expression = replace_expression_vars(expression, self.alternate_names) + if expression.startswith('/'): + return self.get_default_name(expression) or expression + expression = replace_defaults(expression, self.combined) + expression = replace_or_expressions(expression, self.combined, self.datasets, {}) + ids = find_identifiers(expression) + if not ids: + return expression + default_ids = {name: self.get_default_name(name) or name for name in ids} + re_ids = re.compile("|".join(default_ids)) + return re_ids.sub(lambda m: default_ids[m.group(0)], expression) + def get_path(self, name_or_path) -> str | None: """Return hdf path of object in HdfMap""" if name_or_path in self.datasets or name_or_path in self.groups: diff --git a/src/hdfmap/reloader_class.py b/src/hdfmap/reloader_class.py index 9ac9448..e2c0ec1 100644 --- a/src/hdfmap/reloader_class.py +++ b/src/hdfmap/reloader_class.py @@ -186,7 +186,7 @@ def generate_expression(self, expression: str, default=DEFAULT, hdf_file=hdf, expression=expression, default=default, - local_data=self._local_data, + local_data=self._local_data, # doesn't update local data prefer_local=prefer_local, ) diff --git a/tests/test_hdfmap_class.py b/tests/test_hdfmap_class.py index 3e7da16..1bf4448 100644 --- a/tests/test_hdfmap_class.py +++ b/tests/test_hdfmap_class.py @@ -72,6 +72,14 @@ def test_get_default_name(hdf_map): assert hdf_map.get_default_name('measurement_sum') == 'sum' +def test_merge_default_names(hdf_map): + assert hdf_map.merge_default_names('measurement_kth') == 'kth' + assert hdf_map.merge_default_names('(asdf|ddfj|kphi)') == 'kphi' + assert hdf_map.merge_default_names('(asdf|ddfj|bob?(22))') == '22' + assert hdf_map.merge_default_names('mirrors_m2pitch, roi2_h') == 'm2pitch, h' + assert hdf_map.merge_default_names('/entry1/pil3_100k/sum') == 'sum' + + def test_get_group_path(hdf_map): assert hdf_map.get_group_path('sum') == '/entry1/pil3_100k' diff --git a/tests/test_nexus.py b/tests/test_nexus.py index 32b4b07..ae09c75 100644 --- a/tests/test_nexus.py +++ b/tests/test_nexus.py @@ -127,6 +127,10 @@ def test_generate_ids(hdf_map): assert expression == 'signal/Transmission' +def test_merge_default_names(hdf_map): + assert hdf_map.merge_default_names('signal/gains_atten_Transmission') == 'rc/Transmission' + + def test_info_nexus(hdf_map): info = hdf_map.info_nexus() assert 'NXmx: [\'/entry\']' in info