Skip to content

Commit 6d163ef

Browse files
sawenzelclaude
sawenzel
and
claude
committed
Run the alternative-reco step of anchorMC in a subshell
This fixes a problem in the 2-tag setup of anchorMC.sh and anchorMC_DataEmbedding.sh. - The MC software environment was stashed with 'module save' and brought back with 'module restore'. - 'module restore' fails for a module whose version contains '@', as gpu-system did since alidist of 2026-09-05. - ONNXRuntime depends on gpu-system, so O2, O2Physics, QualityControl and O2PDPSuite were not restored. - o2dpg_sim_workflow_anchored.py then failed with "o2 has no attribute ccdb" and no AO2D was produced. - The alternative-reco step now runs in a subshell, so the MC environment is never modified and nothing needs restoring. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent fd75cee commit 6d163ef

2 files changed

Lines changed: 92 additions & 135 deletions

File tree

‎MC/run/ANCHOR/anchorMC.sh‎

Lines changed: 47 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,8 @@ fi
180180

181181
export > env_base.env
182182

183-
# The 2-tag / alternative-reco setup below uses 'module save/purge/restore' to stash
184-
# and swap the software environment. If the shell provides no 'module' function, build
183+
# The 2-tag / alternative-reco setup below uses 'module purge' to swap the software
184+
# environment inside a subshell. If the shell provides no 'module' function, build
185185
# one on a 'modulecmd' binary. Its location varies by version, so resolve it flexibly
186186
# (was hardwired to /usr/bin/modulecmd) and verify it supports saved collections. Fail
187187
# hard -- but only when an alternative-reco tag is set, since single-tag jobs never call
@@ -237,61 +237,61 @@ if [ ! "${MODULEPATH}" ]; then
237237
fi
238238

239239
#<----- START OF part that should run under a clean alternative software environment if this was given ------
240-
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
241-
if [ "${LOADEDMODULES}" ]; then
242-
printenv > env_before_stashing.printenv
243-
echo "Stashing initial modules"
244-
module save initial_modules.list # we stash the current modules environment
245-
module list --no-pager
246-
module purge --no-pager
247-
printenv > env_after_stashing.printenv
248-
echo "Modules after purge"
249-
module list --no-pager
240+
# It runs in a subshell, so the MC software environment of this shell stays untouched
241+
# and does not need to be restored afterwards.
242+
(
243+
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
244+
if [ "${LOADEDMODULES}" ]; then
245+
echo "Unloading initial modules"
246+
module purge --no-pager
247+
fi
248+
echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
249+
/cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
250+
source async_environment.env
251+
export > env_async.env
250252
fi
251-
echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
252-
/cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
253-
source async_environment.env
254-
export > env_async.env
255-
fi
256253

257-
# default async_pass.sh script
258-
DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh
259-
# default destenv_extra.sh script
260-
DPGSETENV=$O2DPG_ROOT/DATA/production/configurations/asyncReco/setenv_extra.sh
254+
# default async_pass.sh script
255+
DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh
256+
# default destenv_extra.sh script
257+
DPGSETENV=$O2DPG_ROOT/DATA/production/configurations/asyncReco/setenv_extra.sh
261258

262-
# a specific async_pass.sh script is in the current directory, assume that one should be used
263-
if [[ -f async_pass.sh ]]; then
264-
# the default is executable, however, this may not be, so make it so
265-
chmod +x async_pass.sh
266-
DPGRECO=./async_pass.sh
267-
else
268-
cp -v $DPGRECO .
269-
fi
259+
# a specific async_pass.sh script is in the current directory, assume that one should be used
260+
if [[ -f async_pass.sh ]]; then
261+
# the default is executable, however, this may not be, so make it so
262+
chmod +x async_pass.sh
263+
DPGRECO=./async_pass.sh
264+
else
265+
cp -v $DPGRECO .
266+
fi
270267

271-
# if there is no setenv_extra.sh in this directory (so no special version is "shipped" with this rpodcution), copy the default one
272-
if [[ ! -f setenv_extra.sh ]] ; then
273-
cp ${DPGSETENV} .
274-
echo_info "Use default setenv_extra.sh from ${DPGSETENV}."
275-
else
276-
echo_info "setenv_extra.sh was found in the current working directory, use it."
277-
fi
268+
# if there is no setenv_extra.sh in this directory (so no special version is "shipped" with this rpodcution), copy the default one
269+
if [[ ! -f setenv_extra.sh ]] ; then
270+
cp ${DPGSETENV} .
271+
echo_info "Use default setenv_extra.sh from ${DPGSETENV}."
272+
else
273+
echo_info "setenv_extra.sh was found in the current working directory, use it."
274+
fi
278275

279-
chmod u+x setenv_extra.sh
276+
chmod u+x setenv_extra.sh
280277

281-
echo_info "Setting up DPGRECO to ${DPGRECO}"
278+
echo_info "Setting up DPGRECO to ${DPGRECO}"
282279

283-
# take out line running the workflow (if we don't have data input)
284-
[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh
280+
# take out line running the workflow (if we don't have data input)
281+
[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh
285282

286-
# create workflow ---> creates the file that can be parsed
287-
export IGNORE_EXISTING_SHMFILES=1
288-
touch list.list
283+
# create workflow ---> creates the file that can be parsed
284+
export IGNORE_EXISTING_SHMFILES=1
285+
touch list.list
289286

290-
# run the async_pass.sh and store output to log file for later inspection and extraction of information
291-
./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log
292-
RECO_RC=$?
287+
# run the async_pass.sh and store output to log file for later inspection and extraction of information
288+
./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log
289+
RECO_RC=$?
293290

294-
echo_info "async_pass.sh finished with ${RECO_RC}"
291+
echo_info "async_pass.sh finished with ${RECO_RC}"
292+
exit ${RECO_RC}
293+
)
294+
RECO_RC=$?
295295

296296
if [[ "${RECO_RC}" != "0" ]] ; then
297297
exit ${RECO_RC}
@@ -305,29 +305,6 @@ fi
305305

306306
export ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP
307307
echo_info "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG"
308-
309-
# get rid of the temporary software environment
310-
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
311-
module purge --no-pager
312-
# restore the initial software environment
313-
echo "Restoring initial environment"
314-
module --no-pager restore initial_modules.list
315-
module saverm initial_modules.list
316-
317-
# Restore overwritten O2DPG variables set by modules but changed by user
318-
# (in particular custom O2DPG_ROOT and O2DPG_MC_CONFIG_ROOT)
319-
# We must avoid piping into a while loop (otherwise the internal export is executed in sub-shell)
320-
printenv > env_after_restore.printenv
321-
while read -r var; do
322-
b=$(grep "^$var=" env_before_stashing.printenv | cut -d= -f2-)
323-
a=$(grep "^$var=" env_after_restore.printenv | cut -d= -f2-)
324-
[[ "$b" != "$a" ]] && export "$var=$b" && echo "Reapplied: $var to ${b}"
325-
done < <(
326-
comm -12 \
327-
<(grep '^O2DPG' env_before_stashing.printenv | cut -d= -f1 | sort) \
328-
<(grep '^O2DPG' env_after_restore.printenv | cut -d= -f1 | sort))
329-
330-
fi
331308
#<----- END OF part that should run under a clean alternative software environment if this was given ------
332309

333310
# now create the local MC config file --> config-json.json

‎MC/run/ANCHOR/anchorMC_DataEmbedding.sh‎

Lines changed: 45 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -196,61 +196,61 @@ if [ ! "${MODULEPATH}" ]; then
196196
fi
197197

198198
#<----- START OF part that should run under a clean alternative software environment if this was given ------
199-
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
200-
if [ "${LOADEDMODULES}" ]; then
201-
printenv > env_before_stashing.printenv
202-
echo "Stashing initial modules"
203-
module save initial_modules.list # we stash the current modules environment
204-
module list --no-pager
205-
module purge --no-pager
206-
printenv > env_after_stashing.printenv
207-
echo "Modules after purge"
208-
module list --no-pager
199+
# It runs in a subshell, so the MC software environment of this shell stays untouched
200+
# and does not need to be restored afterwards.
201+
(
202+
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
203+
if [ "${LOADEDMODULES}" ]; then
204+
echo "Unloading initial modules"
205+
module purge --no-pager
206+
fi
207+
echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
208+
/cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
209+
source async_environment.env
210+
export > env_async.env
209211
fi
210-
echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC"
211-
/cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env
212-
source async_environment.env
213-
export > env_async.env
214-
fi
215212

216-
# default async_pass.sh script
217-
DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh
218-
# default destenv_extra.sh script
219-
DPGSETENV=$O2DPG_ROOT/DATA/production/configurations/asyncReco/setenv_extra.sh
213+
# default async_pass.sh script
214+
DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh
215+
# default destenv_extra.sh script
216+
DPGSETENV=$O2DPG_ROOT/DATA/production/configurations/asyncReco/setenv_extra.sh
220217

221-
# a specific async_pass.sh script is in the current directory, assume that one should be used
222-
if [[ -f async_pass.sh ]]; then
223-
# the default is executable, however, this may not be, so make it so
224-
chmod +x async_pass.sh
225-
DPGRECO=./async_pass.sh
226-
else
227-
cp -v $DPGRECO .
228-
fi
218+
# a specific async_pass.sh script is in the current directory, assume that one should be used
219+
if [[ -f async_pass.sh ]]; then
220+
# the default is executable, however, this may not be, so make it so
221+
chmod +x async_pass.sh
222+
DPGRECO=./async_pass.sh
223+
else
224+
cp -v $DPGRECO .
225+
fi
229226

230-
# if there is no setenv_extra.sh in this directory (so no special version is "shipped" with this rpodcution), copy the default one
231-
if [[ ! -f setenv_extra.sh ]] ; then
232-
cp ${DPGSETENV} .
233-
echo_info "Use default setenv_extra.sh from ${DPGSETENV}."
234-
else
235-
echo_info "setenv_extra.sh was found in the current working directory, use it."
236-
fi
227+
# if there is no setenv_extra.sh in this directory (so no special version is "shipped" with this rpodcution), copy the default one
228+
if [[ ! -f setenv_extra.sh ]] ; then
229+
cp ${DPGSETENV} .
230+
echo_info "Use default setenv_extra.sh from ${DPGSETENV}."
231+
else
232+
echo_info "setenv_extra.sh was found in the current working directory, use it."
233+
fi
237234

238-
chmod u+x setenv_extra.sh
235+
chmod u+x setenv_extra.sh
239236

240-
echo_info "Setting up DPGRECO to ${DPGRECO}"
237+
echo_info "Setting up DPGRECO to ${DPGRECO}"
241238

242-
# take out line running the workflow (if we don't have data input)
243-
[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh
239+
# take out line running the workflow (if we don't have data input)
240+
[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh
244241

245-
# create workflow ---> creates the file that can be parsed
246-
export IGNORE_EXISTING_SHMFILES=1
247-
touch list.list
242+
# create workflow ---> creates the file that can be parsed
243+
export IGNORE_EXISTING_SHMFILES=1
244+
touch list.list
248245

249-
# run the async_pass.sh and store output to log file for later inspection and extraction of information
250-
./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log
251-
RECO_RC=$?
246+
# run the async_pass.sh and store output to log file for later inspection and extraction of information
247+
./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log
248+
RECO_RC=$?
252249

253-
echo_info "async_pass.sh finished with ${RECO_RC}"
250+
echo_info "async_pass.sh finished with ${RECO_RC}"
251+
exit ${RECO_RC}
252+
)
253+
RECO_RC=$?
254254

255255
if [[ "${RECO_RC}" != "0" ]] ; then
256256
exit ${RECO_RC}
@@ -264,26 +264,6 @@ fi
264264

265265
export ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP
266266
echo_info "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG"
267-
268-
# get rid of the temporary software environment
269-
if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then
270-
module purge --no-pager
271-
# restore the initial software environment
272-
echo "Restoring initial environment"
273-
module --no-pager restore initial_modules.list
274-
module saverm initial_modules.list
275-
276-
# Restore overwritten O2DPG variables set by modules but changed by user
277-
# (in particular custom O2DPG_ROOT and O2DPG_MC_CONFIG_ROOT)
278-
printenv > env_after_restore.printenv
279-
comm -12 <(grep '^O2DPG' env_before_stashing.printenv | cut -d= -f1 | sort) \
280-
<(grep '^O2DPG' env_after_restore.printenv | cut -d= -f1 | sort) |
281-
while read -r var; do
282-
b=$(grep "^$var=" env_before_stashing.printenv | cut -d= -f2-)
283-
a=$(grep "^$var=" env_after_restore.printenv | cut -d= -f2-)
284-
[[ "$b" != "$a" ]] && export "$var=$b" && echo "Reapplied: $var to ${b}"
285-
done
286-
fi
287267
#<----- END OF part that should run under a clean alternative software environment if this was given ------
288268

289269
# now create the local MC config file --> config-json.json

0 commit comments

Comments
 (0)