From 6761d9c33d1052f7f0decd82061983d768ff3332 Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Thu, 24 Sep 2026 16:04:52 +0100 Subject: [PATCH 1/6] Remove pytest requirement for run_small_volume --- tests/helpers.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 5c98429d..a731f777 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,8 +5,6 @@ import os import subprocess -import pytest - webstorage_location = "https://ftp.strw.leidenuniv.nl/mcgibbon/SOAP/" test_data_dir = "test_data/" @@ -65,6 +63,8 @@ def requires(filepaths, comm=None): def dont_call_test(func): def empty(*args, **kwargs): + import pytest + return pytest.skip() return empty @@ -89,6 +89,11 @@ def final_test(): # Download the data required for run_small_volume.sh # Call @requires by passing a dummy function dummy = lambda x: x - requires("swift_output/fof_output_0018.hdf5")(dummy)() - requires("swift_output/snap_0018.hdf5")(dummy)() - requires("HBT_output/018/SubSnap_018.0.hdf5")(dummy)() + for filepath in [ + "swift_output/fof_output_0018.hdf5", + "swift_output/snap_0018.hdf5", + "HBT_output/018/SubSnap_018.0.hdf5", + ]: + requires(filepath)(dummy) + if not os.path.exists(f"{test_data_dir}{os.path.basename(filepath)}"): + raise RuntimeError(f"Unable to download {webstorage_location}{filepath}") From e0d8b4156f59a4c8e59cb09cdad497e4fa2f1bf2 Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Thu, 24 Sep 2026 16:13:02 +0100 Subject: [PATCH 2/6] Update install instructions --- README.md | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0e03285e..5510542e 100644 --- a/README.md +++ b/README.md @@ -14,19 +14,32 @@ Please cite SOAP using the ## Installation -The code is written in python and uses mpi4py for parallelism. -IO is also intended to run in parallel, and so -[parallel h5py](https://docs.h5py.org/en/stable/mpi.html) is recommended. -SOAP and its dependencies can be -installed directly using the command -`pip install git+https://github.com/SWIFTSIM/SOAP.git` -but this may install a serial version of h5py. Therefore the following -steps are recommended for install +The code is written in python and uses mpi4py for parallelism. Whichever +install method you use, you will need MPI available so that mpi4py can be +installed. + +### Quick install (serial HDF5) + +SOAP and its dependencies can be installed directly using the command +``` +pip install git+https://github.com/SWIFTSIM/SOAP.git +``` +This will usually install a serial version of h5py. SOAP will run correctly, +but for large simulations the I/O will be slower. + +### Recommended install (parallel HDF5) + +For large runs, [parallel h5py](https://docs.h5py.org/en/stable/mpi.html) is +recommended so that I/O is carried out in parallel. This requires an HDF5 +library which was itself built with MPI support. h5py must then be built from +source against that library before installing SOAP: ``` pip install mpi4py export HDF5_MPI="ON"; export CC=mpicc; pip install --no-binary=h5py h5py pip install git+https://github.com/SWIFTSIM/SOAP.git ``` +If SOAP (and therefore serial h5py) is already installed, then add the flags +`--no-cache-dir` and `--force-reinstall` when reinstalling h5py. ### Installation on COSMA @@ -38,7 +51,7 @@ you can install an SOAP virtual environment by running The command `./tests/run_small_volume.sh` will download a small example simulation, run the group membership and halo properties scripts on it. -This uses the parameter file at `./tests/run_small_volume.yml`, and the +This uses the parameter file at `./tests/small_volume.yml`, and the resulting catalogue is placed in the `output` directory. It also generates the pdf documentation to describe the output file (which is written to `documentation/SOAP.pdf`). @@ -56,7 +69,7 @@ the snapshot number, and a parameter file. For example: ``` snapnum=0077 sim=L1000N0900/DMO_FIDUCIAL -mpirun python python SOAP/group_membership.py \ +mpirun python SOAP/group_membership.py \ --sim-name=${sim} --snap-nr=${snapnum} parameter_files/FLAMINGO.yml ``` @@ -197,12 +210,13 @@ the job name with the slurm sbatch -J flag. ## Modifying the code -You can install an editable version of SOAP by cloning this repository and running: +You can install an editable version of SOAP by cloning this repository and +following the [installation](#installation) steps above, but replacing the +final command with ``` -pip install mpi4py -export HDF5_MPI="ON"; export CC=mpicc; pip install --no-binary=h5py h5py -pip install -e . +pip install -e ".[test]" ``` +This also installs the optional dependencies required to run the tests. The property calculations are defined in the following files in the `SOAP/particle_selection` directory: @@ -217,7 +231,7 @@ Adding new quantities to already defined SOAP apertures is relatively easy. Ther * Next you have to add the quantity to the type of aperture you want it to be calculated for (`aperture_properties.py`, `SO_properties.py`, `subhalo_properties.py`, or `projected_aperture_properties.py`). In all these files there is a class named `property_list` which defines the subset of all properties that are calculated for this specific aperture. * To calculate your quantity you have to define a `@lazy_property` with the same name in the `XXParticleData` class in the same file. There should be a lot of examples of different quantities that are already calculated. An important thing to note is that fields that are used for multiple calculations should have their own `@lazy_property` to avoid loading things multiple times, so check if the things that you need are already there. * Add the property to the parameter file. - * At this point everything should now work. To test the newly added quantities you can run a unit test using `pytest -W error -m pytest tests/test_{NAME_OF_FILE}`. This checks whether the code crashes, and whether there are problems with units and overflows. This should make sure that SOAP never crashes while calculating the new properties. + * At this point everything should now work. To test the newly added quantities you can run a unit test using `pytest -W error tests/test_{NAME_OF_FILE}.py`. This checks whether the code crashes, and whether there are problems with units and overflows. This should make sure that SOAP never crashes while calculating the new properties. If SOAP does crash while evaluating your new property it will try to output the ID of the halo it was processing when it crashed. Then you From 425f8726ed662857e227dc2faed7139df913117b Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Thu, 24 Sep 2026 17:05:01 +0100 Subject: [PATCH 3/6] Add opt-in properites --- README.md | 6 ++++ SOAP/compute_halo_properties.py | 1 + SOAP/core/parameter_file.py | 52 +++++++++++++++++++++++++++++++++ SOAP/property_table.py | 22 ++++++++++++++ parameter_files/README.md | 2 +- tests/test_parameter_file.py | 35 ++++++++++++++++++++++ 6 files changed, 117 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5510542e..a1ad40a3 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,12 @@ Adding new quantities to already defined SOAP apertures is relatively easy. Ther * Add the property to the parameter file. * At this point everything should now work. To test the newly added quantities you can run a unit test using `pytest -W error tests/test_{NAME_OF_FILE}.py`. This checks whether the code crashes, and whether there are problems with units and overflows. This should make sure that SOAP never crashes while calculating the new properties. +If your property is expensive to compute, or requires additional dependencies, +set `opt_in_reason` in its `SOAP/property_table.py` entry to a short (20 +characters or less) reason. It will then only be calculated if it is explicitly +enabled in the parameter file. Any additional dependencies must be imported +within the `@lazy_property`, not at the top of the file. + If SOAP does crash while evaluating your new property it will try to output the ID of the halo it was processing when it crashed. Then you can re-run that halo on a single MPI rank in the python debugger as diff --git a/SOAP/compute_halo_properties.py b/SOAP/compute_halo_properties.py index 4b0d832f..fd75339d 100644 --- a/SOAP/compute_halo_properties.py +++ b/SOAP/compute_halo_properties.py @@ -523,6 +523,7 @@ def compute_halo_properties(): print("Storing processing time for each property") parameter_file.print_unregistered_properties(halo_prop_list, dmo=args.dmo) parameter_file.print_skipped_properties(halo_prop_list, dmo=args.dmo) + parameter_file.print_optin_skipped_properties(halo_prop_list, dmo=args.dmo) parameter_file.print_invalid_properties(halo_prop_list) parameter_file.print_variation_warnings() if not parameter_file.renclose_enabled(): diff --git a/SOAP/core/parameter_file.py b/SOAP/core/parameter_file.py index 4e8d392a..6b0d8a88 100644 --- a/SOAP/core/parameter_file.py +++ b/SOAP/core/parameter_file.py @@ -136,6 +136,12 @@ def __init__( # Only used when calculate_missing_properties is True. self.skipped_properties = set() + # Properties which are not in the parameter file, and which are not + # calculated because they are flagged as opt-in in the property table. + # Stored as {property name: reason}. Only used when + # calculate_missing_properties is True. + self.optin_skipped_properties = {} + # Properties which are enabled in the parameter file, but which cannot # be calculated because the input files lack the datasets they need. self.uncomputable_properties = {} @@ -228,6 +234,16 @@ def missing_datasets(self, property_name: str) -> List[str]: missing.append(dataset_name) return missing + def _opt_in_reason(self, property_name: str): + """ + Get the reason a property is only calculated when explicitly enabled + in the parameter file, or None if it is not an opt-in property. + """ + prop = _property_by_name(property_name) + if prop is None: + return None + return prop.opt_in_reason + def _filter_property_names(self) -> set: """ Get the names of the properties used by the filters defined in the @@ -304,6 +320,18 @@ def get_property_filters(self, base_halo_type: str, full_list: List[str]) -> Dic # Property is not listed in the parameter file for this base_halo_type elif not self.calculate_missing_properties(): filters[property] = False + elif self._opt_in_reason(property) is not None: + # Opt-in properties must be explicitly enabled in the parameter file + if property in self._filter_property_names(): + raise ValueError( + f"{property} is used by a filter, but is an opt-in property " + f'("{self._opt_in_reason(property)}") and so is not calculated ' + f"unless it is explicitly enabled. Please enable it in the " + f'"{base_halo_type}" section of the parameter file.' + ) + filters[property] = False + listed[property] = False + self.optin_skipped_properties[property] = self._opt_in_reason(property) elif missing and property not in self._filter_property_names(): # The property was not asked for explicitly and cannot be # computed, so it is skipped. Properties used by a filter are @@ -409,6 +437,30 @@ def print_skipped_properties(self, halo_prop_list=None, dmo: bool = False) -> No for property in sorted(skipped): print(f" {property}") + def print_optin_skipped_properties( + self, halo_prop_list=None, dmo: bool = False + ) -> None: + """ + Print a list of the properties which are not in the parameter file, and + which are not calculated because they are flagged as opt-in in the + property table, along with the reason for each one. + """ + skipped = dict(self.optin_skipped_properties) + + # In a DMO run, drop properties that would be skipped anyway + # because they are not DMO properties + if dmo and halo_prop_list is not None: + for name in self._non_dmo_property_names(halo_prop_list): + skipped.pop(name, None) + + if len(skipped): + print( + "Not computing the following properties for the reason given, " + "they must be explicitly enabled in the parameter file:" + ) + for property in sorted(skipped): + print(f" {property.ljust(40)}{skipped[property]}") + def print_uncomputable_properties(self) -> None: """ Print a list of the properties which are enabled in the parameter file, diff --git a/SOAP/property_table.py b/SOAP/property_table.py index 07c9970a..5c1ce372 100644 --- a/SOAP/property_table.py +++ b/SOAP/property_table.py @@ -143,6 +143,10 @@ class Property: # Name of the snapshot particle dataset (e.g. "Luminosities") # whose NamedColumns entry should be copied for this property columns_from_snapshot: str = None + # If set, the property is only calculated when it is explicitly enabled in the + # parameter file, even if calculate_missing_properties is True. The value is a + # short reason (<= 20 chars), which is printed when the property is skipped. + opt_in_reason: str = None class PropertyTable: @@ -739,6 +743,7 @@ class PropertyTable: particle_properties=["PartType1/Coordinates", "PartType1/Masses"], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "DarkMatterInertiaTensorReduced": Property( name="DarkMatterInertiaTensorReduced", @@ -751,6 +756,7 @@ class PropertyTable: particle_properties=["PartType1/Coordinates", "PartType1/Masses"], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "DarkMatterInertiaTensorNoniterative": Property( name="DarkMatterInertiaTensorNoniterative", @@ -1342,6 +1348,7 @@ class PropertyTable: particle_properties=["PartType0/Coordinates", "PartType0/Masses"], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "GasInertiaTensorReduced": Property( name="GasInertiaTensorReduced", @@ -1354,6 +1361,7 @@ class PropertyTable: particle_properties=["PartType0/Coordinates", "PartType0/Masses"], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "GasInertiaTensorNoniterative": Property( name="GasInertiaTensorNoniterative", @@ -2218,6 +2226,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "ProjectedTotalInertiaTensorReduced": Property( name="ProjectedTotalInertiaTensorReduced", @@ -2239,6 +2248,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "ProjectedTotalInertiaTensorNoniterative": Property( name="ProjectedTotalInertiaTensorNoniterative", @@ -2293,6 +2303,7 @@ class PropertyTable: particle_properties=["PartType0/Coordinates", "PartType0/Masses"], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "ProjectedGasInertiaTensorReduced": Property( name="ProjectedGasInertiaTensorReduced", @@ -2305,6 +2316,7 @@ class PropertyTable: particle_properties=["PartType0/Coordinates", "PartType0/Masses"], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "ProjectedGasInertiaTensorNoniterative": Property( name="ProjectedGasInertiaTensorNoniterative", @@ -2341,6 +2353,7 @@ class PropertyTable: particle_properties=["PartType4/Coordinates", "PartType4/Masses"], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "ProjectedStellarInertiaTensorReduced": Property( name="ProjectedStellarInertiaTensorReduced", @@ -2353,6 +2366,7 @@ class PropertyTable: particle_properties=["PartType4/Coordinates", "PartType4/Masses"], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "ProjectedStellarInertiaTensorNoniterative": Property( name="ProjectedStellarInertiaTensorNoniterative", @@ -2393,6 +2407,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "ProjectedStellarInertiaTensorReducedLuminosityWeighted": Property( name="ProjectedStellarInertiaTensorReducedLuminosityWeighted", @@ -2409,6 +2424,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted": Property( name="ProjectedStellarInertiaTensorNoniterativeLuminosityWeighted", @@ -2477,6 +2493,7 @@ class PropertyTable: particle_properties=["PartType4/Coordinates", "PartType4/Masses"], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "StellarInertiaTensorReduced": Property( name="StellarInertiaTensorReduced", @@ -2489,6 +2506,7 @@ class PropertyTable: particle_properties=["PartType4/Coordinates", "PartType4/Masses"], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "StellarInertiaTensorNoniterative": Property( name="StellarInertiaTensorNoniterative", @@ -2529,6 +2547,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "StellarInertiaTensorReducedLuminosityWeighted": Property( name="StellarInertiaTensorReducedLuminosityWeighted", @@ -2545,6 +2564,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "StellarInertiaTensorNoniterativeLuminosityWeighted": Property( name="StellarInertiaTensorNoniterativeLuminosityWeighted", @@ -2795,6 +2815,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=2, + opt_in_reason="Expensive to compute", ), "TotalInertiaTensorReduced": Property( name="TotalInertiaTensorReduced", @@ -2816,6 +2837,7 @@ class PropertyTable: ], output_physical=True, a_scale_exponent=0, + opt_in_reason="Expensive to compute", ), "TotalInertiaTensorNoniterative": Property( name="TotalInertiaTensorNoniterative", diff --git a/parameter_files/README.md b/parameter_files/README.md index c10b5439..5228438c 100644 --- a/parameter_files/README.md +++ b/parameter_files/README.md @@ -247,7 +247,7 @@ defined_constants: Contains information about how to run SOAP -- **calculate_missing_properties**: Optional, default True. If set to true then SOAP will calculate any properties which are not listed in the parameter file, provided the input files contain the datasets those properties require. Properties which cannot be calculated are skipped, and are listed at the start of the run. If set to false then SOAP will ignore any property which is not listed in the parameter file. +- **calculate_missing_properties**: Optional, default True. If set to true then SOAP will calculate any properties which are not listed in the parameter file, provided the input files contain the datasets those properties require. Properties which cannot be calculated are skipped, and are listed at the start of the run. If set to false then SOAP will ignore any property which is not listed in the parameter file. Some properties are flagged as opt-in in the property table (e.g. because they are expensive to compute, or require additional dependencies). These are never calculated unless they are explicitly enabled in the parameter file, and are listed at the start of the run along with the reason they were skipped. - **reduced_snapshots**: Optional. We create reduced snapshots where we keep the particles within the virial radius of certain objects. The values here determine which halos to keep. - **min_halo_mass**: The minimumum M200 halo mass to keep diff --git a/tests/test_parameter_file.py b/tests/test_parameter_file.py index ef7a15cf..1fc228ad 100644 --- a/tests/test_parameter_file.py +++ b/tests/test_parameter_file.py @@ -2,10 +2,12 @@ import glob import os +from types import SimpleNamespace import pytest from SOAP.core.parameter_file import ParameterFile +from SOAP.property_table import PropertyTable REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -378,3 +380,36 @@ def test_absent_particle_type_is_not_treated_as_missing(): assert filters["DustMass"] == "basic" assert pf.skipped_properties == set() assert pf.uncomputable_properties == {} + + +def test_opt_in_reasons_are_short(): + # The reason is printed in a column when the property is skipped + for prop in PropertyTable.full_property_list.values(): + if prop.opt_in_reason is not None: + assert len(prop.opt_in_reason) <= 20, ( + f'The opt_in_reason "{prop.opt_in_reason}" for {prop.name} is ' + f"longer than 20 characters. Reasons are printed in a column " + f"next to the property name when properties are skipped, so " + f"they must be short to keep the output readable." + ) + + +def test_non_dmo_opt_in_properties_are_not_printed_for_dmo_run(monkeypatch, capsys): + # GasMass is not a DMO property, TotalMass is. Flag both as opt-in. + props = { + prop.name: prop + for prop in PropertyTable.full_property_list.values() + if prop.name in ("GasMass", "TotalMass") + } + for prop in props.values(): + monkeypatch.setattr(prop, "opt_in_reason", "expensive") + + pf = make_parameter_file(section={"properties": {}, "variations": VARIATIONS}) + filters = pf.get_property_filters("ApertureProperties", ["GasMass", "TotalMass"]) + assert filters == {"GasMass": False, "TotalMass": False} + + halo_prop_list = [SimpleNamespace(property_list=props)] + pf.print_optin_skipped_properties(halo_prop_list, dmo=True) + out = capsys.readouterr().out + assert "TotalMass" in out + assert "GasMass" not in out From 9792be122c710f7bebc56393de0368499eba4c12 Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Fri, 25 Sep 2026 11:04:14 +0100 Subject: [PATCH 4/6] Update documentation for different halo finders --- README.md | 3 +- SOAP/catalogue_readers/read_hbtplus.py | 25 +++- SOAP/core/combine_chunks.py | 1 + SOAP/core/halo_centres.py | 7 +- SOAP/core/parameter_file.py | 60 ++++++++-- SOAP/core/soap_args.py | 1 + SOAP/group_membership.py | 8 +- parameter_files/COLIBRE_HYBRID.yml | 4 - parameter_files/COLIBRE_THERMAL.yml | 4 - parameter_files/FLAMINGO.yml | 12 +- parameter_files/README.md | 15 +-- parameter_files/halo_finders.md | 154 +++++++++++++++++++++++++ tests/small_volume.yml | 2 - 13 files changed, 251 insertions(+), 45 deletions(-) create mode 100644 parameter_files/halo_finders.md diff --git a/README.md b/README.md index a1ad40a3..dbc66807 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,8 @@ the halo finder to use, which halo definitions to use, and which properties to calculate for each halo definition. A description of all possible fields can be found in [`parameter_files/README.md`](parameter_files/README.md), alongside a number -of example parameter files. +of example parameter files. How to specify each of the supported halo finders is +described in [`parameter_files/halo_finders.md`](parameter_files/halo_finders.md). ### Compression diff --git a/SOAP/catalogue_readers/read_hbtplus.py b/SOAP/catalogue_readers/read_hbtplus.py index 6e0e8b65..3fac8668 100644 --- a/SOAP/catalogue_readers/read_hbtplus.py +++ b/SOAP/catalogue_readers/read_hbtplus.py @@ -14,13 +14,20 @@ def hbt_filename(hbt_basename, file_nr): return f"{hbt_basename}.{file_nr}.hdf5" -def read_hbtplus_groupnr(basename, read_potential_energies=False, registry=None): +def read_hbtplus_groupnr( + basename, read_potential_energies=False, registry=None, index_by_track_id=False +): """ Read HBTplus output and return group number for each particle ID Potential energies will not be returned by default. To return the potential energies a unit registry must be passed. + If index_by_track_id is True then the group number of each particle is the + TrackId of its subhalo, rather than the position of the subhalo in the + catalogue. This only has an effect for unsorted catalogues, since for + sorted catalogues the position is already equal to the TrackId. + """ from mpi4py import MPI @@ -106,6 +113,7 @@ def read_hbtplus_groupnr(basename, read_potential_energies=False, registry=None) # Number of particles in each subhalo halo_size = halos["Nbound"] + halo_track_id = halos["TrackId"] del halos # Apply same combination process to potential energies @@ -152,6 +160,8 @@ def read_hbtplus_groupnr(basename, read_potential_energies=False, registry=None) total_nr_halos = comm.allreduce(nr_local_halos) halo_offset = comm.scan(len(halo_size), op=MPI.SUM) - len(halo_size) halo_index = np.arange(nr_local_halos, dtype=int) + halo_offset + if index_by_track_id and not sorted_file: + halo_index = halo_track_id grnr_bound = np.repeat(halo_index, halo_size) # Assign ranking by binding energy to the particles @@ -182,7 +192,13 @@ def read_hbtplus_groupnr(basename, read_potential_energies=False, registry=None) def read_hbtplus_catalogue( - comm, basename, a_unit, registry, boxsize, keep_orphans=False + comm, + basename, + a_unit, + registry, + boxsize, + keep_orphans=False, + index_by_track_id=False, ): """ Read in the HBTplus halo catalogue, distributed over communicator comm. @@ -192,6 +208,9 @@ def read_hbtplus_catalogue( a_unit - unyt a factor registry - unyt unit registry boxsize - box size as a unyt quantity + index_by_track_id - use the TrackId as the index of each halo. This only + has an effect for unsorted catalogues, since for sorted + catalogues the index is already equal to the TrackId Returns a dict of unyt arrays with the halo properies. Arrays which must always be returned: @@ -304,6 +323,8 @@ def read_hbtplus_catalogue( nr_local_halos = len(keep) local_offset = comm.scan(nr_local_halos) - nr_local_halos index = np.arange(nr_local_halos, dtype=int) + local_offset + if index_by_track_id and not sorted_file: + index = subhalo["TrackId"] index = index[keep] index = unyt.unyt_array( index, units=unyt.dimensionless, dtype=int, registry=registry diff --git a/SOAP/core/combine_chunks.py b/SOAP/core/combine_chunks.py index 733fd0d0..1d975896 100644 --- a/SOAP/core/combine_chunks.py +++ b/SOAP/core/combine_chunks.py @@ -773,6 +773,7 @@ def combine_chunks( cellgrid.a_unit, cellgrid.snap_unit_registry, cellgrid.boxsize, + index_by_track_id=args.index_by_track_id, ) prev_order, _ = spatial_sort( prev_data["cofp"], diff --git a/SOAP/core/halo_centres.py b/SOAP/core/halo_centres.py index a5538c75..f289d4ba 100644 --- a/SOAP/core/halo_centres.py +++ b/SOAP/core/halo_centres.py @@ -78,7 +78,12 @@ def __init__( ) elif args.halo_format == "HBTplus": halo_data = read_hbtplus.read_hbtplus_catalogue( - comm, halo_basename, a_unit, registry, boxsize + comm, + halo_basename, + a_unit, + registry, + boxsize, + index_by_track_id=args.index_by_track_id, ) elif args.halo_format == "Subfind": halo_data = read_subfind.read_gadget4_catalogue( diff --git a/SOAP/core/parameter_file.py b/SOAP/core/parameter_file.py index 6b0d8a88..704abaa7 100644 --- a/SOAP/core/parameter_file.py +++ b/SOAP/core/parameter_file.py @@ -37,21 +37,32 @@ def _property_by_name(name: str): return _PROPERTY_BY_NAME.get(name) +# Keys in the HaloFinder section (other than type and filename) which each halo +# finder supports. A warning is printed if a key is set for a halo finder which +# does not support it. Add a new halo finder here, and document it in +# parameter_files/halo_finders.md. +_HALO_FINDER_KEYS = { + "HBTplus": { + "fof_filename", + "fof_radius_filename", + "read_potential_energies", + "index_by_track_id", + }, + "VR": set(), + "Subfind": set(), + "SubfindEagle": set(), + "Rockstar": set(), +} + # Known parameter file structure, used by check_schema to flag typos. A value # of None means the keys directly under that section are user-named or free-form # and are not checked; a set lists the only keys allowed directly under that # section. Add a key here when a new option is introduced. _ALLOWED_KEYS = { "Parameters": None, - "Snapshots": {"filename", "fof_filename"}, - "HaloFinder": { - "type", - "filename", - "fof_filename", - "fof_radius_filename", - "read_potential_energies", - }, - "GroupMembership": {"filename"}, + "Snapshots": {"filename"}, + "HaloFinder": {"type", "filename"}.union(*_HALO_FINDER_KEYS.values()), + "GroupMembership": {"filename", "fof_ids_filename"}, "ExtraInput": None, "HaloProperties": {"filename", "chunk_dir"}, "SubhaloProperties": {"properties"}, @@ -73,6 +84,21 @@ def _property_by_name(name: str): } +def halo_finder_warnings(halo_finder: Dict) -> List[str]: + """ + Return a warning for each key in the HaloFinder section which is not + supported by the chosen halo finder type, and so will be ignored. + """ + supported = _HALO_FINDER_KEYS.get(halo_finder.get("type"), set()) + optional_keys = _ALLOWED_KEYS["HaloFinder"] - {"type", "filename"} + return [ + f'Warning: "HaloFinder/{key}" is not supported for halo finder ' + f'"{halo_finder.get("type")}" and will be ignored' + for key in halo_finder + if key in optional_keys and key not in supported + ] + + class ParameterFile: """ Internal representation of the parameter file. @@ -609,12 +635,22 @@ def print_variation_warnings(self) -> None: def check_schema(self) -> None: """ - Abort if the parameter file has an unrecognised section, or a mistyped + Abort if the parameter file has an unrecognised section, a mistyped key directly under a section which has a fixed set of keys (see - _ALLOWED_KEYS). This catches typos which would otherwise be silently - ignored. It does not check value types, or keys nested more deeply. + _ALLOWED_KEYS), or an unknown halo finder type. This catches typos which + would otherwise be silently ignored. It does not check value types, or + keys nested more deeply. Also warns about HaloFinder keys which the + chosen halo finder does not support. """ errors = [] + halo_finder = self.parameters.get("HaloFinder", {}) + if "type" in halo_finder and halo_finder["type"] not in _HALO_FINDER_KEYS: + errors.append( + f'unknown halo finder type "{halo_finder["type"]}", the supported ' + f"types are {', '.join(_HALO_FINDER_KEYS)}" + ) + for warning in halo_finder_warnings(halo_finder): + print(warning) for section, block in self.parameters.items(): if section not in _ALLOWED_KEYS: errors.append(f'unknown section "{section}"') diff --git a/SOAP/core/soap_args.py b/SOAP/core/soap_args.py index 4f1015fa..3ea6bbdb 100644 --- a/SOAP/core/soap_args.py +++ b/SOAP/core/soap_args.py @@ -220,6 +220,7 @@ def get_soap_args(comm): args.read_potential_energies = all_args["HaloFinder"].get( "read_potential_energies", False ) + args.index_by_track_id = all_args["HaloFinder"].get("index_by_track_id", False) args.fof_group_filename = all_args["HaloFinder"].get("fof_filename", "") args.fof_radius_filename = all_args["HaloFinder"].get("fof_radius_filename", "") args.output_file = all_args["HaloProperties"]["filename"] diff --git a/SOAP/group_membership.py b/SOAP/group_membership.py index 0f03b0ff..f6637690 100644 --- a/SOAP/group_membership.py +++ b/SOAP/group_membership.py @@ -12,6 +12,7 @@ from virgo.util.partial_formatter import PartialFormatter from SOAP.core import combine_args, swift_units +from SOAP.core.parameter_file import halo_finder_warnings from SOAP.catalogue_readers import read_vr from SOAP.catalogue_readers import read_hbtplus from SOAP.catalogue_readers import read_subfind @@ -178,17 +179,20 @@ def main(): # Extract parameters we need snap_nr = args["Parameters"]["snap_nr"] - fof_filename = args["Snapshots"].get("fof_filename", "") + fof_filename = args["GroupMembership"].get("fof_ids_filename", "") swift_filename = args["Snapshots"]["filename"] halo_format = args["HaloFinder"]["type"] halo_basename = args["HaloFinder"]["filename"] read_potential_energies = args["HaloFinder"].get("read_potential_energies", False) + index_by_track_id = args["HaloFinder"].get("index_by_track_id", False) output_filename = args["GroupMembership"]["filename"] if comm_rank == 0: print(f"Input snapshot is {swift_filename}") print(f"Halo basename is {halo_basename}") print(f"Snapshot number is {snap_nr}") + for warning in halo_finder_warnings(args["HaloFinder"]): + print(warning) # Substitute in the snapshot number where necessary pf = PartialFormatter() @@ -238,6 +242,7 @@ def main(): halo_basename, read_potential_energies=True, registry=registry, + index_by_track_id=index_by_track_id, ) ) else: @@ -246,6 +251,7 @@ def main(): total_nr_halos, ids_bound, grnr_bound, rank_bound = ( read_hbtplus.read_hbtplus_groupnr( halo_basename, + index_by_track_id=index_by_track_id, ) ) potential_energies = None diff --git a/parameter_files/COLIBRE_HYBRID.yml b/parameter_files/COLIBRE_HYBRID.yml index aeef96a8..8cba1769 100644 --- a/parameter_files/COLIBRE_HYBRID.yml +++ b/parameter_files/COLIBRE_HYBRID.yml @@ -16,10 +16,6 @@ HaloFinder: fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" fof_radius_filename: "{sim_dir}/{sim_name}/fof_radii/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" read_potential_energies: true - #type: VR - #filename: "{sim_dir}/halo_{snap_nr:04d}" - #type: Subfind - #filename: "{sim_dir}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" GroupMembership: # Where to write the group membership files diff --git a/parameter_files/COLIBRE_THERMAL.yml b/parameter_files/COLIBRE_THERMAL.yml index c89f5b22..94fc7a74 100644 --- a/parameter_files/COLIBRE_THERMAL.yml +++ b/parameter_files/COLIBRE_THERMAL.yml @@ -16,10 +16,6 @@ HaloFinder: fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" fof_radius_filename: "{sim_dir}/{sim_name}/fof_radii/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" read_potential_energies: true - #type: VR - #filename: "{sim_dir}/halo_{snap_nr:04d}" - #type: Subfind - #filename: "{sim_dir}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" GroupMembership: # Where to write the group membership files diff --git a/parameter_files/FLAMINGO.yml b/parameter_files/FLAMINGO.yml index dd214e46..47504e20 100644 --- a/parameter_files/FLAMINGO.yml +++ b/parameter_files/FLAMINGO.yml @@ -10,25 +10,19 @@ Parameters: Snapshots: # Use {snap_nr:04d} for the snapshot number and {file_nr} for the file number. filename: "{sim_dir}/{sim_name}/snapshots/flamingo_{snap_nr:04d}/flamingo_{snap_nr:04d}.{file_nr}.hdf5" - # Pass the location of snapshots with FOF IDs if the FOF has been re-run - # If fof_filename is not passed the FOF IDs in the snapshot will be used - fof_filename: "/cosma8/data/dp004/jlvc76/FLAMINGO/FOF/{sim_name}/fof_snapshot/flamingo_{snap_nr:04d}/flamingo_{snap_nr:04d}.{file_nr}.hdf5" # Which halo finder we're using, and base name for halo finder output files HaloFinder: - #type: VR - #filename: "{sim_dir}/{sim_name}/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}" type: HBTplus filename: "{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}" fof_filename: "/cosma8/data/dp004/jlvc76/FLAMINGO/FOF/{sim_name}/fof_catalog/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" - #type: Subfind - #filename: "{sim_dir}/{sim_name}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" - #type: Rockstar - #filename: "{sim_dir}/{sim_name}/Rockstar/snapshot_{snap_nr:04d}/halos_{snap_nr:04d}" GroupMembership: # Where to write the group membership files filename: "{sim_dir}/{sim_name}/SOAP-HBT/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5" + # Pass the location of snapshots with FOF IDs if the FOF has been re-run + # If fof_ids_filename is not passed the FOF IDs in the snapshot will be used + fof_ids_filename: "/cosma8/data/dp004/jlvc76/FLAMINGO/FOF/{sim_name}/fof_snapshot/flamingo_{snap_nr:04d}/flamingo_{snap_nr:04d}.{file_nr}.hdf5" ExtraInput: xray_filename: "{output_dir}/{sim_name}/recalculated_xray/xray_{snap_nr:04d}/xray_{snap_nr:04d}.{file_nr}.hdf5" diff --git a/parameter_files/README.md b/parameter_files/README.md index 5228438c..79405812 100644 --- a/parameter_files/README.md +++ b/parameter_files/README.md @@ -28,7 +28,6 @@ Alongside these values you must pass `--sim-name` as an argument when running SO This section defines the format and location of the SWIFT simulation snapshots. **If there are chunk files you must pass `{file_nr}` in the filepath template**, SOAP does not currently read virtual hdf5 files. - **filename**: Template for the snapshot files. Use `{snap_nr:04d}` for the snapshot number and `{file_nr}` for the file number, e.g. `"{sim_dir}/{sim_name}/snapshots/flamingo_{snap_nr:04d}/flamingo_{snap_nr:04d}.{file_nr}.hdf5"` -- **fof_filename**: Optional. Path to snapshots with FOF (Friends-of-Friends) IDs if FOF has been re-run. This is only used by the `group_membership.py` script as the values are then stored in the membership files. ### Extra input @@ -39,15 +38,12 @@ If a dataset is present in both the snapshot and the extra input files, the valu ### Halo Finder -Settings for the halo finding algorithm and output file locations. +Settings for the input halo catalogue. -- **type**: The subhalo finder being used. Possible options are `HBTplus`, `VR`, `Subfind`, and `Rockstar`. -- **filename**: Template for input halo catalogue files. The format of this depends on the halo finder as they each have a different output structure. - - HBTplus: `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}"` - - Sorted HBTplus: `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/OrderedSubSnap_{snap_nr:03d}.hdf5"` -- **fof_filename**: Template for FOF catalog files. Used for storing host FOF information for central subhalos. Only supported for HBTplus -- **fof_radius_filename**: Template for FOF catalog files which contain the "Groups/Radii" dataset. These were produced by a post-processing script, and are missing from the main FOFs -- **read_potential_energies**: Optional boolean value, defaults to False. Whether to read potential energies and place them in the membership files. Only supported for HBTplus +- **type**: The subhalo finder being used. Possible options are `HBTplus`, `VR`, `Subfind`, `SubfindEagle`, and `Rockstar`. +- **filename**: Template for input halo catalogue files. The format of this depends on the halo finder. + +Some halo finders support additional keys. See [`halo_finders.md`](halo_finders.md) for how to specify each halo finder, and the additional keys each one supports. ### Group Membership @@ -55,6 +51,7 @@ Settings for the halo finding algorithm and output file locations. Configuration for writing group membership files. - **filename**: Template for group membership file paths, e.g. `"{output_dir}/{sim_name}/SOAP_uncompressed/membership_{snap_nr:04d}/membership_{snap_nr:04d}.{file_nr}.hdf5"` +- **fof_ids_filename**: Optional. Template for snapshots with FOF (Friends-of-Friends) IDs, if FOF has been re-run. The FOF IDs are read from these files instead of the original snapshots, and are stored in the membership files. ### Halo Properties diff --git a/parameter_files/halo_finders.md b/parameter_files/halo_finders.md new file mode 100644 index 00000000..bf272819 --- /dev/null +++ b/parameter_files/halo_finders.md @@ -0,0 +1,154 @@ +# Halo finders + +SOAP reads the output of a halo finder to decide which particles belong to +each subhalo (when creating the membership files) and where each subhalo is +centred (when computing halo properties). The halo finder is set in the +`HaloFinder` section of the parameter file. Every halo finder needs two keys: + +- **type**: Which halo finder produced the catalogue. One of the section + names below. +- **filename**: Template for the halo catalogue files. What this should point to + depends on the halo finder, as described below. Use `{snap_nr}` for the snapshot + number, e.g. `{snap_nr:03d}`. + +Some halo finders support extra keys, which are listed in their section. If a +key is set which the chosen halo finder does not support then SOAP prints a +warning and ignores it. + +The value of `type` can be used as `{halo_finder}` in the filepath templates in +other sections of the parameter file. + +### HBTplus + +Catalogues from [HBT-HERONS](https://github.com/SWIFTSIM/HBT-HERONS). +Both the unsorted catalogues which HBT outputs directly, and catalogues +which have been sorted by TrackId into a single file, are supported. + +- **filename**: + - Unsorted catalogues: The path to the `SubSnap` files without the `.N.hdf5` + suffix, e.g. `"{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}"` + - Sorted catalogues: The full path to the sorted file, e.g. + `"{sim_dir}/{sim_name}/HBT/OrderedSubSnap_{snap_nr:03d}.hdf5"` + + SOAP uses the unsorted catalogues if `{filename}.0.hdf5` exists, + otherwise it assumes `filename` is a sorted catalogue. +- **fof_filename**: Optional. Template for the FOF catalogue files, e.g. + `"{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}.hdf5"`. + If this is set, the centre, mass and size of the host FOF group are + written to the output for central subhalos. +- **fof_radius_filename**: Optional. Template for FOF catalogue files which + contain the `Groups/Radii` dataset, which is missing from the original FOF + catalogues. These files can be created with `misc/calculate_fof_radii.py`. + Requires `fof_filename` to be set. +- **read_potential_energies**: Optional, defaults to `false`. Whether to read the + potential energies of the bound particles from the HBT catalogues and write + them to the membership files. +- **index_by_track_id**: Optional, defaults to `false`. By default the index of + each subhalo (`GroupNr_bound` in the membership files, and + `InputHalos/HaloCatalogueIndex` in the SOAP catalogue) is its position in the + HBT catalogue. If this is `true` then the TrackId of the subhalo is used + as its index. This only affects unsorted catalogues, since in the sorted + catalogues the position of each subhalo is already equal to its TrackId. The + same value must be used when creating the membership files and when running + SOAP. + +HBT catalogues contain every subhalo which has been identified existed in +the simulation, so every TrackId is present at each snapshot. +This includes orphan subhalos, +which have been disrupted and have no bound particles. SOAP only computes +properties for resolved subhalos (those with bound particles), so in general +the SOAP catalogue has fewer entries than the input HBT catalogue. + +If the catalogues for the previous and next snapshots exist then +SOAP also computes `SOAP/ProgenitorIndex` and `SOAP/DescendantIndex`. + +``` +HaloFinder: + type: HBTplus + filename: "{sim_dir}/{sim_name}/HBT/{snap_nr:03d}/SubSnap_{snap_nr:03d}" + fof_filename: "{sim_dir}/{sim_name}/fof/fof_output_{snap_nr:04d}/fof_output_{snap_nr:04d}.{file_nr}.hdf5" + read_potential_energies: true +``` + +### VR + +Catalogues from [VELOCIraptor](https://github.com/ICRAR/VELOCIraptor-STF). + +- **filename**: The path to the catalogue files without the `.properties`, + `.catalog_groups`, etc. suffixes, e.g. + `"{sim_dir}/{sim_name}/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}"`. + Both single file and multi-file (`.properties.N`) output are supported. + +``` +HaloFinder: + type: VR + filename: "{sim_dir}/{sim_name}/VR/catalogue_{snap_nr:04d}/vr_catalogue_{snap_nr:04d}" +``` + +### Subfind + +Catalogues from the Gadget-4 version of Subfind. + +- **filename**: The path to the Gadget-4 group sorted snapshot files, without the + `.N.hdf5` suffix, e.g. `"{sim_dir}/{sim_name}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}"`. + The filename must end with the three digit snapshot number. The group + catalogues are then expected to be in + `groups_{snap_nr:03d}/fof_subhalo_tab_{snap_nr:03d}.N.hdf5`, in the same + directory as `snapdir_{snap_nr:03d}`. + +``` +HaloFinder: + type: Subfind + filename: "{sim_dir}/{sim_name}/snapdir_{snap_nr:03d}/snapshot_{snap_nr:03d}" +``` + +### SubfindEagle + +Subfind catalogues from the original EAGLE simulations. The `group_membership.py` +script does not support these catalogues. Instead, `misc/convert_eagle.py` +converts the EAGLE snapshots to SWIFT format and creates the membership +files at the same time. See the documentation at the top of that script. + +- **filename**: The path to the `subfind_tab` files without the `.N.hdf5` suffix, e.g. + `"{sim_dir}/{sim_name}/subfind/groups_{snap_nr:03d}/subfind_tab_{snap_nr:03d}"` + +``` +HaloFinder: + type: SubfindEagle + filename: "{sim_dir}/{sim_name}/subfind/groups_{snap_nr:03d}/subfind_tab_{snap_nr:03d}" +``` + +### Rockstar + +Catalogues from [Rockstar](https://bitbucket.org/gfcstanford/rockstar). + +- **filename**: The path to the Rockstar binary files without the `.N.bin` + suffix, e.g. `"{sim_dir}/{sim_name}/Rockstar/snapshot_{snap_nr:04d}/halos_{snap_nr:04d}"`. + The filename must end with the four digit snapshot number. The files + `merger_tree/snapshot_{snap_nr:04d}/parents_{snap_nr:04d}.N.list` must also + exist, where `merger_tree` is in the same directory as `snapshot_{snap_nr:04d}`. + +``` +HaloFinder: + type: Rockstar + filename: "{sim_dir}/{sim_name}/Rockstar/snapshot_{snap_nr:04d}/halos_{snap_nr:04d}" +``` + +**Caveat:** Rockstar does not work well with SOAP, as SOAP assumes that each +particle is bound to at most one subhalo. Rockstar assigns mass inclusively, +meaning a particle can be bound to a subhalo and to its parent at the same time. +Its output files also don't record which particles are bound to each halo, only +the particles associated with it. As a result, properties which depend on bound +membership are not reliable for Rockstar, whereas properties which use all +particles (e.g. spherical overdensities) are unaffected. See +[Forouhar Moreno et al. 2025](https://arxiv.org/abs/2502.06932), §3.4 and §3.6. + +### Adding a new halo finder + +To add support for a new halo finder: + +- Write a reader in `SOAP/catalogue_readers/`. +- Call it from `SOAP/group_membership.py` and `SOAP/core/halo_centres.py`. +- Add the halo finder, along with any extra keys it supports, to + `_HALO_FINDER_KEYS` in `SOAP/core/parameter_file.py`. +- Add a section to this file. diff --git a/tests/small_volume.yml b/tests/small_volume.yml index 9f74b547..6e6594cc 100644 --- a/tests/small_volume.yml +++ b/tests/small_volume.yml @@ -10,8 +10,6 @@ HaloFinder: type: HBTplus filename: "{sim_dir}/SubSnap_{snap_nr:03d}" fof_filename: "{sim_dir}/fof_output_{snap_nr:04d}.hdf5" - # type: VR - # filename: "{sim_dir}/vr_{snap_nr:03d}" GroupMembership: filename: "{output_dir}/membership_{snap_nr:04d}/membership_{snap_nr:04d}.hdf5" From 4da4bb7614ea71be57a4a52ba5f492edf909d96f Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Fri, 25 Sep 2026 11:29:26 +0100 Subject: [PATCH 5/6] Add note on why index_by_track_id is useful --- parameter_files/halo_finders.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parameter_files/halo_finders.md b/parameter_files/halo_finders.md index bf272819..95dc05ea 100644 --- a/parameter_files/halo_finders.md +++ b/parameter_files/halo_finders.md @@ -52,6 +52,15 @@ which have been sorted by TrackId into a single file, are supported. same value must be used when creating the membership files and when running SOAP. + This is useful because creating the membership files requires the IDs of the + particles in each subhalo. The unsorted catalogues contain these, and sorted + catalogues can too, but storing them in both duplicates a lot of data. The + suggested workflow is to create the membership files and run SOAP using the + unsorted catalogues with `index_by_track_id: true`, and then generate sorted + catalogues without particle IDs. SOAP can later be rerun using the sorted + catalogues and the existing membership files, since both use the TrackId + as the subhalo index. + HBT catalogues contain every subhalo which has been identified existed in the simulation, so every TrackId is present at each snapshot. This includes orphan subhalos, From a9162719bcca1b83bdee64e9288d193821993860 Mon Sep 17 00:00:00 2001 From: robjmcgibbon Date: Fri, 25 Sep 2026 12:20:22 +0100 Subject: [PATCH 6/6] Enable opt-in properites for tests --- parameter_files/halo_finders.md | 2 +- tests/dummy_halo_generator.py | 14 ++++++++++++++ tests/test_SO_properties.py | 7 +++++-- tests/test_aperture_properties.py | 7 +++++-- tests/test_projected_aperture_properties.py | 9 +++++++-- tests/test_subhalo_properties.py | 7 +++++-- 6 files changed, 37 insertions(+), 9 deletions(-) diff --git a/parameter_files/halo_finders.md b/parameter_files/halo_finders.md index 95dc05ea..50ed492e 100644 --- a/parameter_files/halo_finders.md +++ b/parameter_files/halo_finders.md @@ -61,7 +61,7 @@ which have been sorted by TrackId into a single file, are supported. catalogues and the existing membership files, since both use the TrackId as the subhalo index. -HBT catalogues contain every subhalo which has been identified existed in +HBT catalogues contain every subhalo which has been identified in the simulation, so every TrackId is present at each snapshot. This includes orphan subhalos, which have been disrupted and have no bound particles. SOAP only computes diff --git a/tests/dummy_halo_generator.py b/tests/dummy_halo_generator.py index e2903cd6..4fcf4397 100644 --- a/tests/dummy_halo_generator.py +++ b/tests/dummy_halo_generator.py @@ -26,6 +26,20 @@ from SOAP.particle_filter.cold_dense_gas_filter import ColdDenseGasFilter +def opt_in_properties(property_list: Dict) -> Dict[str, bool]: + """ + Return a properties dictionary for the parameter file which explicitly + enables every opt-in property in property_list. Opt-in properties are + not computed by calculate_missing_properties, so tests which check all + properties must enable them. + """ + return { + prop.name: True + for prop in property_list.values() + if prop.opt_in_reason is not None + } + + class DummySnapshot: """ Dummy SWIFT snapshot. Can be used to replace an actual snapshot in diff --git a/tests/test_SO_properties.py b/tests/test_SO_properties.py index 59fcf428..44abb5da 100644 --- a/tests/test_SO_properties.py +++ b/tests/test_SO_properties.py @@ -10,7 +10,7 @@ RadiusMultipleSOProperties, ) -from dummy_halo_generator import DummyHaloGenerator +from dummy_halo_generator import DummyHaloGenerator, opt_in_properties def test_SO_properties_random_halo(): @@ -33,7 +33,10 @@ def test_SO_properties_random_halo(): "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", "PartType0/XrayLuminositiesRestframe": "PartType0/XrayLuminositiesRestframe", "PartType0/XrayPhotonLuminositiesRestframe": "PartType0/XrayPhotonLuminositiesRestframe", - } + }, + "SOProperties": { + "properties": opt_in_properties(SOProperties.property_list) + }, } ) dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( diff --git a/tests/test_aperture_properties.py b/tests/test_aperture_properties.py index 5e481d71..afe7020b 100644 --- a/tests/test_aperture_properties.py +++ b/tests/test_aperture_properties.py @@ -13,7 +13,7 @@ InclusiveSphereProperties, ) -from dummy_halo_generator import DummyHaloGenerator +from dummy_halo_generator import DummyHaloGenerator, opt_in_properties def test_aperture_properties(): @@ -41,7 +41,10 @@ def test_aperture_properties(): "aliases": { "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } + }, + "ApertureProperties": { + "properties": opt_in_properties(ExclusiveSphereProperties.property_list) + }, } ) dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( diff --git a/tests/test_projected_aperture_properties.py b/tests/test_projected_aperture_properties.py index 62bbabce..f6711399 100644 --- a/tests/test_projected_aperture_properties.py +++ b/tests/test_projected_aperture_properties.py @@ -8,7 +8,7 @@ ProjectedApertureProperties, ) -from dummy_halo_generator import DummyHaloGenerator +from dummy_halo_generator import DummyHaloGenerator, opt_in_properties def test_projected_aperture_properties(): @@ -31,7 +31,12 @@ def test_projected_aperture_properties(): "aliases": { "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } + }, + "ProjectedApertureProperties": { + "properties": opt_in_properties( + ProjectedApertureProperties.property_list + ) + }, } ) dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases( diff --git a/tests/test_subhalo_properties.py b/tests/test_subhalo_properties.py index 5acde0cd..67d15528 100644 --- a/tests/test_subhalo_properties.py +++ b/tests/test_subhalo_properties.py @@ -7,7 +7,7 @@ from SOAP.property_calculation.stellar_age_calculator import StellarAgeCalculator from SOAP.particle_selection.subhalo_properties import SubhaloProperties -from dummy_halo_generator import DummyHaloGenerator +from dummy_halo_generator import DummyHaloGenerator, opt_in_properties def test_subhalo_properties(): @@ -31,7 +31,10 @@ def test_subhalo_properties(): "aliases": { "PartType0/ElementMassFractions": "PartType0/SmoothedElementMassFractions", "PartType4/ElementMassFractions": "PartType4/SmoothedElementMassFractions", - } + }, + "SubhaloProperties": { + "properties": opt_in_properties(SubhaloProperties.property_list) + }, } ) dummy_halos.get_cell_grid().snapshot_datasets.setup_aliases(