Skip to content

Commit decdb6f

Browse files
Matt-CarreMatt-Carre
committed
feat(example): adds additional visr recon workflow example
1 parent f8f38cd commit decdb6f

6 files changed

Lines changed: 1292 additions & 2 deletions

File tree

‎src/copier_template/pyproject.toml.jinja‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ description = "Run tests with coverage"
160160
commands = [
161161
[
162162
"pytest",
163-
"--cov=python_interface_to_workflows",
163+
"--cov={% endraw %}{{repo_name}}{% raw %}",
164164
"--cov-report",
165165
"term",
166166
"--cov-report",

‎src/copier_template/src/{{ project_name }}/templates/visr_recon.txt.jinja‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
{% raw %}
2+
apiVersion: argoproj.io/v1alpha1
3+
kind: WorkflowTemplate
4+
metadata:
5+
name: visr-recon-with-{% endraw %}{{repo_name}}{% raw %}
6+
annotations:
7+
workflows.argoproj.io/description: |-
8+
ViSR recon
9+
example.yaml
10+
workflows.argoproj.io/title: ViSR recon
11+
workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}
12+
labels:
13+
workflows.diamond.ac.uk/science-group-imaging: 'true'
14+
spec:
15+
entrypoint: workflowentry
16+
templates:
17+
- name: workflowentry
18+
dag:
19+
tasks:
20+
- name: tomo-recon
21+
template: tomo-recon
22+
arguments:
23+
parameters:
24+
- name: config
25+
value: |-
26+
[
27+
{
28+
"method": "standard_tomo",
29+
"module_path": "httomo.data.hdf.loaders",
30+
"parameters": {
31+
"data_path": "/entry1/tomo_entry/data/data",
32+
"image_key_path": "/entry1/tomo_entry/instrument/detector/image_key",
33+
"rotation_angles": {
34+
"data_path": "/entry1/tomo_entry/data/rotation_angle"
35+
},
36+
"preview": {
37+
"detector_y": {
38+
"start": 100,
39+
"stop": 102
40+
}
41+
}
42+
}
43+
},
44+
{
45+
"method": "remove_outlier",
46+
"module_path": "tomopy.misc.corr",
47+
"parameters": {
48+
"dif": 0.1,
49+
"size": 3,
50+
"axis": "auto"
51+
}
52+
},
53+
{
54+
"method": "dark_flat_field_correction",
55+
"module_path": "httomolibgpu.prep.normalize",
56+
"parameters": {
57+
"flats_multiplier": 1,
58+
"darks_multiplier": 1
59+
}
60+
},
61+
{
62+
"method": "find_center_vo",
63+
"module_path": "httomolibgpu.recon.rotation",
64+
"parameters": {
65+
"ind": null,
66+
"smin": -50,
67+
"smax": 50,
68+
"srad": 6,
69+
"step": 0.25,
70+
"ratio": 0.5,
71+
"drop": 20
72+
},
73+
"id": "centering",
74+
"side_outputs": {
75+
"cor": "centre_of_rotation"
76+
}
77+
},
78+
{
79+
"method": "FBP3d_tomobar",
80+
"module_path": "httomolibgpu.recon.algorithm",
81+
"parameters": {
82+
"center": "${{centering.side_outputs.centre_of_rotation}}",
83+
"filter_freq_cutoff": 0.6,
84+
"recon_size": null,
85+
"recon_mask_radius": null
86+
},
87+
"save_result": true
88+
},
89+
{
90+
"method": "calculate_stats",
91+
"module_path": "httomo.methods",
92+
"parameters": {},
93+
"id": "statistics",
94+
"side_outputs": {
95+
"glob_stats": "glob_stats"
96+
}
97+
},
98+
{
99+
"method": "rescale_to_int",
100+
"module_path": "httomolib.misc.rescale",
101+
"parameters": {
102+
"perc_range_min": 0,
103+
"perc_range_max": 100,
104+
"bits": 8,
105+
"glob_stats": "${{statistics.side_outputs.glob_stats}}"
106+
},
107+
"save_result": true
108+
}
109+
]
110+
- name: input
111+
value: /dls/i12/data/2025/cm40628-3/rawdata/188700.nxs
112+
- name: output
113+
value: ''
114+
- name: recon_outdir_name
115+
value: sweep-run
116+
- name: nprocs
117+
value: '1'
118+
- name: memory
119+
value: 1Gi
120+
- name: convert-recon-data-format
121+
depends: tomo-recon
122+
template: convert-recon-data-format
123+
arguments:
124+
parameters:
125+
- name: recon_dir_path
126+
value: '{{tasks.tomo-recon.outputs.parameters.out-path}}'
127+
- name: tomo-recon
128+
podSpecPatch: '{"containers": [{"name": "main", "resources": {"limits": {"cpu":
129+
"{{inputs.parameters.nprocs}}", "memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu":
130+
"{{inputs.parameters.nprocs}}"}, "requests": {"cpu": "{{inputs.parameters.nprocs}}",
131+
"memory": "{{inputs.parameters.memory}}", "nvidia.com/gpu": "{{inputs.parameters.nprocs}}"}}}]}'
132+
tolerations:
133+
- effect: NoSchedule
134+
key: nvidia.com/gpu
135+
operator: Exists
136+
- effect: NoSchedule
137+
key: nodetype
138+
operator: Equal
139+
value: gpu
140+
- effect: NoSchedule
141+
key: nodegroup
142+
operator: Equal
143+
value: workflows
144+
inputs:
145+
parameters:
146+
- name: config
147+
- name: input
148+
- name: output
149+
- name: recon_outdir_name
150+
- name: nprocs
151+
- name: memory
152+
outputs:
153+
parameters:
154+
- name: out-path
155+
valueFrom:
156+
path: /tmp/parameters.json
157+
script:
158+
image: ghcr.io/diamondlightsource/httomo:latest
159+
source: |-
160+
import os
161+
import sys
162+
sys.path.append(os.getcwd())
163+
import json
164+
try: config = json.loads(r'''{{inputs.parameters.config}}''')
165+
except: config = r'''{{inputs.parameters.config}}'''
166+
try: input = json.loads(r'''{{inputs.parameters.input}}''')
167+
except: input = r'''{{inputs.parameters.input}}'''
168+
try: memory = json.loads(r'''{{inputs.parameters.memory}}''')
169+
except: memory = r'''{{inputs.parameters.memory}}'''
170+
try: nprocs = json.loads(r'''{{inputs.parameters.nprocs}}''')
171+
except: nprocs = r'''{{inputs.parameters.nprocs}}'''
172+
try: output = json.loads(r'''{{inputs.parameters.output}}''')
173+
except: output = r'''{{inputs.parameters.output}}'''
174+
try: recon_outdir_name = json.loads(r'''{{inputs.parameters.recon_outdir_name}}''')
175+
except: recon_outdir_name = r'''{{inputs.parameters.recon_outdir_name}}'''
176+
177+
import json
178+
import subprocess
179+
loaded_config = json.dumps(config)
180+
subprocess.check_call(['/opt/conda/bin/mpirun', '-n', str(nprocs), '/opt/conda/bin/python', '-m', 'httomo', 'run', '--pipeline-format', 'json', '--output-folder-name', recon_outdir_name, input, loaded_config, output])
181+
with open('/tmp/parameters.json', 'w') as f:
182+
json.dump(f'{output}/{recon_outdir_name}', f)
183+
command:
184+
- /opt/conda/bin/python
185+
env:
186+
- name: CUPY_CACHE_DIR
187+
value: /tmp/.cupy/kernel_cache
188+
- name: MKL_NUM_THREADS
189+
value: '1'
190+
- name: NUMEXPR_NUM_THREADS
191+
value: '1'
192+
- name: OMP_NUM_THREADS
193+
value: '1'
194+
volumeMounts:
195+
- name: session
196+
mountPath: '{{workflow.parameters.visitdir}}'
197+
- name: tmpdir
198+
mountPath: /tmp
199+
- name: convert-recon-data-format
200+
inputs:
201+
parameters:
202+
- name: recon_dir_path
203+
- name: tmpdir_path
204+
default: /tmp
205+
- name: raw_recon_filename
206+
default: recon.raw
207+
- name: metadata_filename
208+
default: metadata.json
209+
outputs:
210+
artifacts:
211+
- name: recon
212+
path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.raw_recon_filename}}'
213+
archive:
214+
none: {}
215+
- name: metadata
216+
path: '{{inputs.parameters.tmpdir_path}}/{{inputs.parameters.metadata_filename}}'
217+
archive:
218+
none: {}
219+
script:
220+
image: ghcr.io/diamondlightsource/httomo:latest
221+
source: |-
222+
import os
223+
import sys
224+
sys.path.append(os.getcwd())
225+
import json
226+
try: metadata_filename = json.loads(r'''{{inputs.parameters.metadata_filename}}''')
227+
except: metadata_filename = r'''{{inputs.parameters.metadata_filename}}'''
228+
try: raw_recon_filename = json.loads(r'''{{inputs.parameters.raw_recon_filename}}''')
229+
except: raw_recon_filename = r'''{{inputs.parameters.raw_recon_filename}}'''
230+
try: recon_dir_path = json.loads(r'''{{inputs.parameters.recon_dir_path}}''')
231+
except: recon_dir_path = r'''{{inputs.parameters.recon_dir_path}}'''
232+
try: tmpdir_path = json.loads(r'''{{inputs.parameters.tmpdir_path}}''')
233+
except: tmpdir_path = r'''{{inputs.parameters.tmpdir_path}}'''
234+
235+
import json
236+
from pathlib import Path
237+
import h5py
238+
RAW_RECON_PATH = f'{tmpdir_path}/{raw_recon_filename}'
239+
HDF5_RECON_DIR = Path(recon_dir_path)
240+
HDF5_RECON_FILENAME_PATTERN = '*-httomolib-rescale_to_int.h5'
241+
hdf5_recon_data_path = list(HDF5_RECON_DIR.glob(HDF5_RECON_FILENAME_PATTERN))[0]
242+
with h5py.File(hdf5_recon_data_path, 'r') as f:
243+
data = f['/data'][:]
244+
data.tofile(RAW_RECON_PATH)
245+
METADATA_PATH = f'{tmpdir_path}/{metadata_filename}'
246+
order = 'C' if data.flags.c_contiguous else 'F'
247+
metadata = {'shape': list(data.shape), 'dtype': str(data.dtype), 'order': order}
248+
with open(METADATA_PATH, 'w') as f:
249+
f.write(json.dumps(metadata, indent=2))
250+
command:
251+
- /opt/conda/bin/python
252+
volumeMounts:
253+
- name: session
254+
mountPath: '{{workflow.parameters.visitdir}}'
255+
- name: tmpdir
256+
mountPath: /tmp
257+
volumeClaimTemplates:
258+
- metadata:
259+
name: tmpdir
260+
spec:
261+
accessModes:
262+
- ReadWriteOnce
263+
resources:
264+
requests:
265+
storage: 1Gi
266+
volumes:
267+
- name: session
268+
hostPath:
269+
path: '{{workflow.parameters.visitdir}}'
270+
type: Directory
271+
arguments:
272+
parameters:
273+
- name: visitdir
274+
valueFrom:
275+
configMapKeyRef:
276+
name: sessionspaces
277+
key: data_directory
278+
{% endraw %}

‎src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ from hera.workflows.archive import NoneArchiveStrategy
1818
# and was created with the included dockerfile.
1919
global_config.set_class_defaults( # pyright: ignore
2020
Script,
21-
image="ghcr.io/diamondlightsource/python-interface-to-workflows-mounted-image:latest",
21+
image="ghcr.io/diamondlightsource/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest",
2222
)
2323

2424

0 commit comments

Comments
 (0)