From 2773fd3376067f40cbe7a59a714b51f0ea45a1e1 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 7 Sep 2026 10:22:33 +0100 Subject: [PATCH 1/7] Add transmission in to standard data collection --- helm/daq-config-server/converter_map.yaml | 2 +- .../app/_file_converter_map.py | 4 +- .../models/i15_1/positions_to_times.py | 51 ++++++++++----- .../test_tth_angle_to_collection_time.txt | 28 ++++----- .../models/i15_1/test_positions_to_times.py | 63 ++++++++++--------- 5 files changed, 89 insertions(+), 59 deletions(-) diff --git a/helm/daq-config-server/converter_map.yaml b/helm/daq-config-server/converter_map.yaml index b6586db..909be19 100644 --- a/helm/daq-config-server/converter_map.yaml +++ b/helm/daq-config-server/converter_map.yaml @@ -67,4 +67,4 @@ - path: "/dls_sw/i19-1/software/i19-acquisition/i19-shared/lookup/energy_to_id_gap_look_up_table.txt" converter: UndulatorEnergyGapLookupTable - path: "/dls_sw/i15-1/software/daq_configuration/tth_angle_to_collection_time.json" - converter: AnglesToTimes + converter: CollectionSpecification diff --git a/src/daq_config_server/app/_file_converter_map.py b/src/daq_config_server/app/_file_converter_map.py index 5e63f84..a2b0655 100644 --- a/src/daq_config_server/app/_file_converter_map.py +++ b/src/daq_config_server/app/_file_converter_map.py @@ -13,7 +13,7 @@ from daq_config_server.models.feature_settings.i04_feature_settings import ( I04FeatureSettings, ) -from daq_config_server.models.i15_1.positions_to_times import AnglesToTimes +from daq_config_server.models.i15_1.positions_to_times import CollectionSpecification from daq_config_server.models.i15_1.xpdf_crystal_lut import XpdfCrystalLookupTable from daq_config_server.models.i15_1.xpdf_parameters import TemperatureControllersConfig from daq_config_server.models.lookup_tables import ( @@ -61,7 +61,7 @@ def init_converter_map(config: ConverterConfig): "HyperionFeatureSettings": HyperionFeatureSettings.from_domain_properties, "TemperatureControllersConfig": TemperatureControllersConfig.from_xpdf_parameters, "XpdfCrystalLookupTable": XpdfCrystalLookupTable.from_contents, - "AnglesToTimes": AnglesToTimes.from_lut, + "CollectionSpecification": CollectionSpecification.from_lut, } diff --git a/src/daq_config_server/models/i15_1/positions_to_times.py b/src/daq_config_server/models/i15_1/positions_to_times.py index b026037..c337c27 100644 --- a/src/daq_config_server/models/i15_1/positions_to_times.py +++ b/src/daq_config_server/models/i15_1/positions_to_times.py @@ -1,29 +1,52 @@ -from pydantic import field_validator +from pydantic import BaseModel, field_validator from daq_config_server.models.base_model import ConfigModel from daq_config_server.models.utils import parse_lut_rows -class AnglesToTimes(ConfigModel): - tth_angle_to_collection_time: dict[float, float] +class SpecificationPerPosition(BaseModel): + exposure_time: float + transmission: float - @field_validator("tth_angle_to_collection_time", mode="after") + @field_validator("transmission") + @classmethod + def _validate_transmission(cls, transmission: float) -> float: + allowed_transmissions = [100, 50, 10, 1, 0.1, 0.01, 0.001] + if transmission not in allowed_transmissions: + raise ValueError( + f"Transmission must be one of {sorted(allowed_transmissions)}" + ) + return transmission + + +class CollectionSpecification(ConfigModel): + tth_angle_to_specification: dict[float, SpecificationPerPosition] + + @field_validator("tth_angle_to_specification", mode="after") @classmethod def _normalise_to_fractions( - cls, angles_and_rel_times: dict[float, float] - ) -> dict[float, float]: - """This allows the time values inside the config file to have arbitary units. + cls, + angles_spec: dict[float, SpecificationPerPosition], + ) -> dict[float, SpecificationPerPosition]: + """This allows the time values inside the config file to have arbitrary units. Once parsed, they will all be as a fraction out of 1, so you can do positions_and_rel_times[position] * total_time to get the time for that position. """ - weighting = 1 / sum(angles_and_rel_times.values()) - return { - position: rel_time * weighting - for position, rel_time in angles_and_rel_times.items() - } + weighting = 1 / sum([spec.exposure_time for spec in angles_spec.values()]) + for spec in angles_spec.values(): + spec.exposure_time *= weighting + + return angles_spec @classmethod def from_lut(cls, contents: str): - rows = parse_lut_rows(contents, types=[float, float]) - return cls(tth_angle_to_collection_time={row[0]: row[1] for row in rows}) + rows = parse_lut_rows(contents, types=[float, float, float]) + return cls( + tth_angle_to_specification={ + row[0]: SpecificationPerPosition( + exposure_time=row[1], transmission=row[2] + ) + for row in rows + } + ) diff --git a/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt b/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt index 75208f7..291ba49 100644 --- a/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt +++ b/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt @@ -1,21 +1,21 @@ # This file specifies which angles the tth will go do during a regular data collection. -# It also specifies the proportion of the total collection time spent at each angle. +# It also specifies the proportion of the total collection time spent at each angle and the transmission at each angle. # The time values can be in any arbitary units - their relative values are all that matter. # E.g, the following two tables are equivalent # -# 90 1 -# 180 2 -# 270 3 +# 90 1 100 +# 180 2 100 +# 270 3 100 # # -# 90 0.05 -# 180 0.01 -# 270 0.15 +# 90 0.05 100 +# 180 0.01 100 +# 270 0.15 100 -# Angle (deg) Time -10 0.05 -20 0.05 -30 0.1 -40 0.2 -50 0.3 -60 0.3 +# Angle (deg) Time Transmission +10 0.05 0.01 +20 0.05 0.1 +30 0.1 1 +40 0.2 10 +50 0.3 50 +60 0.3 100 diff --git a/tests/unit_tests/models/i15_1/test_positions_to_times.py b/tests/unit_tests/models/i15_1/test_positions_to_times.py index 0d79519..abb84cf 100644 --- a/tests/unit_tests/models/i15_1/test_positions_to_times.py +++ b/tests/unit_tests/models/i15_1/test_positions_to_times.py @@ -1,44 +1,51 @@ import pytest +from pydantic import ValidationError from tests.constants import TestDataPaths -from daq_config_server.models.i15_1.positions_to_times import AnglesToTimes +from daq_config_server.models.i15_1.positions_to_times import ( + CollectionSpecification, + SpecificationPerPosition, +) def test_positions_to_times_parses_json_contents(): with open(TestDataPaths.TEST_I15_1_POSITIONS_TIMES_LUT) as f: contents = f.read() - result = AnglesToTimes.from_lut(contents) - assert result.tth_angle_to_collection_time == { - 10: 0.05, - 20: 0.05, - 30: 0.1, - 40: 0.2, - 50: 0.3, - 60: 0.3, + result = CollectionSpecification.from_lut(contents) + assert result.tth_angle_to_specification == { + 10: SpecificationPerPosition(exposure_time=0.05, transmission=0.01), + 20: SpecificationPerPosition(exposure_time=0.05, transmission=0.1), + 30: SpecificationPerPosition(exposure_time=0.1, transmission=1), + 40: SpecificationPerPosition(exposure_time=0.2, transmission=10), + 50: SpecificationPerPosition(exposure_time=0.3, transmission=50), + 60: SpecificationPerPosition(exposure_time=0.3, transmission=100), } def test_positions_to_times_normalises_times(): - result = AnglesToTimes( - tth_angle_to_collection_time={ - 10: 1, - 20: 1, - 30: 2, - 40: 4, - 50: 6, - 60: 6, - } - ) - - assert result.tth_angle_to_collection_time == pytest.approx( # type: ignore + result = CollectionSpecification.model_validate( { - 10: 0.05, - 20: 0.05, - 30: 0.1, - 40: 0.2, - 50: 0.3, - 60: 0.3, + "tth_angle_to_specification": { + 10: {"exposure_time": 1, "transmission": 0.01}, + 20: {"exposure_time": 1, "transmission": 0.01}, + 30: {"exposure_time": 2, "transmission": 0.01}, + 40: {"exposure_time": 4, "transmission": 0.01}, + 50: {"exposure_time": 6, "transmission": 0.01}, + 60: {"exposure_time": 6, "transmission": 0.01}, + } } ) - assert sum(result.tth_angle_to_collection_time.values()) == 1 + + exposures = [ + spec.exposure_time for spec in result.tth_angle_to_specification.values() + ] + + assert exposures == pytest.approx([0.05, 0.05, 0.1, 0.2, 0.3, 0.3]) + assert sum(exposures) == 1 + + +@pytest.mark.parametrize("transmission", [2, 0.02, 0.0001, 101]) +def test_positions_to_times_rejects_invalid_transmission(transmission: float): + with pytest.raises(ValidationError): + SpecificationPerPosition(exposure_time=1, transmission=transmission) From f92875b40168356568054a6d56c85d7a07ddd579 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 7 Sep 2026 10:24:09 +0100 Subject: [PATCH 2/7] Rename collection specification --- .../{positions_to_times.py => collection_specification.py} | 0 ...t_positions_to_times.py => test_collection_specification.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/daq_config_server/models/i15_1/{positions_to_times.py => collection_specification.py} (100%) rename tests/unit_tests/models/i15_1/{test_positions_to_times.py => test_collection_specification.py} (96%) diff --git a/src/daq_config_server/models/i15_1/positions_to_times.py b/src/daq_config_server/models/i15_1/collection_specification.py similarity index 100% rename from src/daq_config_server/models/i15_1/positions_to_times.py rename to src/daq_config_server/models/i15_1/collection_specification.py diff --git a/tests/unit_tests/models/i15_1/test_positions_to_times.py b/tests/unit_tests/models/i15_1/test_collection_specification.py similarity index 96% rename from tests/unit_tests/models/i15_1/test_positions_to_times.py rename to tests/unit_tests/models/i15_1/test_collection_specification.py index abb84cf..439144b 100644 --- a/tests/unit_tests/models/i15_1/test_positions_to_times.py +++ b/tests/unit_tests/models/i15_1/test_collection_specification.py @@ -2,7 +2,7 @@ from pydantic import ValidationError from tests.constants import TestDataPaths -from daq_config_server.models.i15_1.positions_to_times import ( +from daq_config_server.models.i15_1.collection_specification import ( CollectionSpecification, SpecificationPerPosition, ) From b9bbe9996ec421b6ec2cb7bce502baf5c9b5b9b5 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 7 Sep 2026 10:25:49 +0100 Subject: [PATCH 3/7] Change filename for collection --- helm/daq-config-server/converter_map.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/daq-config-server/converter_map.yaml b/helm/daq-config-server/converter_map.yaml index 909be19..142bec8 100644 --- a/helm/daq-config-server/converter_map.yaml +++ b/helm/daq-config-server/converter_map.yaml @@ -66,5 +66,5 @@ converter: DetectorXYLookupTable - path: "/dls_sw/i19-1/software/i19-acquisition/i19-shared/lookup/energy_to_id_gap_look_up_table.txt" converter: UndulatorEnergyGapLookupTable -- path: "/dls_sw/i15-1/software/daq_configuration/tth_angle_to_collection_time.json" +- path: "/dls_sw/i15-1/software/daq_configuration/collection_specification.json" converter: CollectionSpecification From db2ac1e9e556da6500996b357d14ef04c3392428 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 7 Sep 2026 10:28:29 +0100 Subject: [PATCH 4/7] Rename more things --- src/daq_config_server/app/_file_converter_map.py | 4 +++- tests/test_data/i15-1/test_tth_angle_to_collection_time.txt | 2 +- .../models/i15_1/test_collection_specification.py | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/daq_config_server/app/_file_converter_map.py b/src/daq_config_server/app/_file_converter_map.py index a2b0655..8853ea0 100644 --- a/src/daq_config_server/app/_file_converter_map.py +++ b/src/daq_config_server/app/_file_converter_map.py @@ -13,7 +13,9 @@ from daq_config_server.models.feature_settings.i04_feature_settings import ( I04FeatureSettings, ) -from daq_config_server.models.i15_1.positions_to_times import CollectionSpecification +from daq_config_server.models.i15_1.collection_specification import ( + CollectionSpecification, +) from daq_config_server.models.i15_1.xpdf_crystal_lut import XpdfCrystalLookupTable from daq_config_server.models.i15_1.xpdf_parameters import TemperatureControllersConfig from daq_config_server.models.lookup_tables import ( diff --git a/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt b/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt index 291ba49..cd17342 100644 --- a/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt +++ b/tests/test_data/i15-1/test_tth_angle_to_collection_time.txt @@ -12,7 +12,7 @@ # 180 0.01 100 # 270 0.15 100 -# Angle (deg) Time Transmission +# Angle (deg) Time Transmission (percentage) 10 0.05 0.01 20 0.05 0.1 30 0.1 1 diff --git a/tests/unit_tests/models/i15_1/test_collection_specification.py b/tests/unit_tests/models/i15_1/test_collection_specification.py index 439144b..5626348 100644 --- a/tests/unit_tests/models/i15_1/test_collection_specification.py +++ b/tests/unit_tests/models/i15_1/test_collection_specification.py @@ -8,7 +8,7 @@ ) -def test_positions_to_times_parses_json_contents(): +def test_collection_spec_parses_json_contents(): with open(TestDataPaths.TEST_I15_1_POSITIONS_TIMES_LUT) as f: contents = f.read() @@ -23,7 +23,7 @@ def test_positions_to_times_parses_json_contents(): } -def test_positions_to_times_normalises_times(): +def test_collection_spec_normalises_times(): result = CollectionSpecification.model_validate( { "tth_angle_to_specification": { @@ -46,6 +46,6 @@ def test_positions_to_times_normalises_times(): @pytest.mark.parametrize("transmission", [2, 0.02, 0.0001, 101]) -def test_positions_to_times_rejects_invalid_transmission(transmission: float): +def test_collection_spec_rejects_invalid_transmission(transmission: float): with pytest.raises(ValidationError): SpecificationPerPosition(exposure_time=1, transmission=transmission) From 47d799c7b4ed626f4580e8b6a923f7096a1956ac Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 7 Sep 2026 15:24:07 +0100 Subject: [PATCH 5/7] Ignore pyright warning --- tests/unit_tests/models/i15_1/test_collection_specification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/models/i15_1/test_collection_specification.py b/tests/unit_tests/models/i15_1/test_collection_specification.py index 5626348..c6697af 100644 --- a/tests/unit_tests/models/i15_1/test_collection_specification.py +++ b/tests/unit_tests/models/i15_1/test_collection_specification.py @@ -41,7 +41,7 @@ def test_collection_spec_normalises_times(): spec.exposure_time for spec in result.tth_angle_to_specification.values() ] - assert exposures == pytest.approx([0.05, 0.05, 0.1, 0.2, 0.3, 0.3]) + assert exposures == pytest.approx([0.05, 0.05, 0.1, 0.2, 0.3, 0.3]) # pyright: ignore[reportUnknownMemberType] assert sum(exposures) == 1 From d51ecb61303382a1f66357e81e40c2f7e278ad20 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 8 Sep 2026 13:30:19 +0100 Subject: [PATCH 6/7] Remove validation of transmissionpre-c --- .../models/i15_1/collection_specification.py | 10 ---------- .../models/i15_1/test_collection_specification.py | 6 ------ 2 files changed, 16 deletions(-) diff --git a/src/daq_config_server/models/i15_1/collection_specification.py b/src/daq_config_server/models/i15_1/collection_specification.py index c337c27..78f8f0b 100644 --- a/src/daq_config_server/models/i15_1/collection_specification.py +++ b/src/daq_config_server/models/i15_1/collection_specification.py @@ -8,16 +8,6 @@ class SpecificationPerPosition(BaseModel): exposure_time: float transmission: float - @field_validator("transmission") - @classmethod - def _validate_transmission(cls, transmission: float) -> float: - allowed_transmissions = [100, 50, 10, 1, 0.1, 0.01, 0.001] - if transmission not in allowed_transmissions: - raise ValueError( - f"Transmission must be one of {sorted(allowed_transmissions)}" - ) - return transmission - class CollectionSpecification(ConfigModel): tth_angle_to_specification: dict[float, SpecificationPerPosition] diff --git a/tests/unit_tests/models/i15_1/test_collection_specification.py b/tests/unit_tests/models/i15_1/test_collection_specification.py index c6697af..9274387 100644 --- a/tests/unit_tests/models/i15_1/test_collection_specification.py +++ b/tests/unit_tests/models/i15_1/test_collection_specification.py @@ -43,9 +43,3 @@ def test_collection_spec_normalises_times(): assert exposures == pytest.approx([0.05, 0.05, 0.1, 0.2, 0.3, 0.3]) # pyright: ignore[reportUnknownMemberType] assert sum(exposures) == 1 - - -@pytest.mark.parametrize("transmission", [2, 0.02, 0.0001, 101]) -def test_collection_spec_rejects_invalid_transmission(transmission: float): - with pytest.raises(ValidationError): - SpecificationPerPosition(exposure_time=1, transmission=transmission) From dce951e48f192b85a35e4df2b2ef58ed62b8d04d Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Tue, 8 Sep 2026 14:11:54 +0100 Subject: [PATCH 7/7] Fix linting --- tests/unit_tests/models/i15_1/test_collection_specification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/models/i15_1/test_collection_specification.py b/tests/unit_tests/models/i15_1/test_collection_specification.py index 9274387..7fca52b 100644 --- a/tests/unit_tests/models/i15_1/test_collection_specification.py +++ b/tests/unit_tests/models/i15_1/test_collection_specification.py @@ -1,5 +1,4 @@ import pytest -from pydantic import ValidationError from tests.constants import TestDataPaths from daq_config_server.models.i15_1.collection_specification import (