From b5d1afd1e32fb23041484cf287d6b87347af4fe7 Mon Sep 17 00:00:00 2001 From: Richard O'Shaughnessy Date: Tue, 11 Aug 2026 19:10:31 -0700 Subject: [PATCH] util_RandomizeOverlapOrder: interleave across worker files, not just within them The draw randomises which n_min points each worker file contributes and their order WITHIN that file, then appends each file's block in file order, so the merged output is [file0][file1]... That is invisible to a consumer that reads the whole file. The nested ILE does not: it reads a prefix. Measured on a live S240629by run, ILE evaluates rows 0-2999 of a 20,000-row merge, so with 25 workers x 800 it saw workers 0-3 and nothing from the other 21; at the more common 6 x 3334 the whole prefix sits inside worker 0. Which is exactly the failure the file header warns about -- "Important when merging files from many workers, to avoid accidentally using only the output from one of them." The hyperpipeline branch of write_joingrids_sub already gets this right, concatenating every shard and piping through shuf with the comment "shuffle so spokes are interleaved". This brings the XML path into line with its sibling rather than introducing a new policy. Verified on 25 real CIP worker files: row count unchanged at 20,000, no duplicates introduced, and contiguous same-worker runs go 25 -> 19,248 with all 25 workers represented in the first 3000 rows instead of 4. --preserve-block-order restores the previous behaviour exactly (25 runs, 4 workers) for reproducing earlier runs. Co-Authored-By: Claude Opus 5 --- .../Code/bin/util_RandomizeOverlapOrder.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py b/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py index 6869ca4e5..632d5c2c7 100755 --- a/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py @@ -28,6 +28,7 @@ optp.add_option("--fref",default=20,type=float,help="Reference frequency. Depending on approximant and age of implementation, may be ignored") optp.add_option("--n-min",default=20,type=int,help="Minimum size of file to include. NOT USED") optp.add_option("--output-file",default='merged_output',type=str,help="Merged output file") +optp.add_option("--preserve-block-order",action='store_true',help="Do NOT interleave across input files; append each file's block in file order. This is the pre-2026-08 behaviour and is only for reproducing runs made before the interleave fix.") optp.add_option("--verbose",action='store_true',help="Print messages") opts, args = optp.parse_args() @@ -61,4 +62,14 @@ P_to_add = [P_list_list[indx][a] for a in indx_to_take] P_list += P_to_add +# Interleave ACROSS files. The draw above randomises only WITHIN each file, and the blocks are +# appended in file order, so the merged output is [file0][file1]...[fileN]. That is invisible to a +# consumer that reads the whole file, but the nested ILE reads only a PREFIX (measured: the first +# 3000 rows), so it saw just the first few workers -- with 25 workers x 800, four of them, and with +# 6 x 3334, only the first. That is precisely the "accidentally using only the output from one of +# them" failure this tool exists to prevent. The hyperpipeline branch of write_joingrids_sub +# already does this ("shuffle so spokes are interleaved"); this brings the XML path into line. +if not opts.preserve_block_order: + P_list = [P_list[k] for k in np.random.permutation(len(P_list))] + lalsimutils.ChooseWaveformParams_array_to_xml(P_list,opts.output_file)