diff --git a/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py b/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py index 6869ca4e5..632d5c2c7 100755 --- a/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py +++ b/MonteCarloMarginalizeCode/Code/bin/util_RandomizeOverlapOrder.py @@ -28,6 +28,7 @@ optp.add_option("--fref",default=20,type=float,help="Reference frequency. Depending on approximant and age of implementation, may be ignored") optp.add_option("--n-min",default=20,type=int,help="Minimum size of file to include. NOT USED") optp.add_option("--output-file",default='merged_output',type=str,help="Merged output file") +optp.add_option("--preserve-block-order",action='store_true',help="Do NOT interleave across input files; append each file's block in file order. This is the pre-2026-08 behaviour and is only for reproducing runs made before the interleave fix.") optp.add_option("--verbose",action='store_true',help="Print messages") opts, args = optp.parse_args() @@ -61,4 +62,14 @@ P_to_add = [P_list_list[indx][a] for a in indx_to_take] P_list += P_to_add +# Interleave ACROSS files. The draw above randomises only WITHIN each file, and the blocks are +# appended in file order, so the merged output is [file0][file1]...[fileN]. That is invisible to a +# consumer that reads the whole file, but the nested ILE reads only a PREFIX (measured: the first +# 3000 rows), so it saw just the first few workers -- with 25 workers x 800, four of them, and with +# 6 x 3334, only the first. That is precisely the "accidentally using only the output from one of +# them" failure this tool exists to prevent. The hyperpipeline branch of write_joingrids_sub +# already does this ("shuffle so spokes are interleaved"); this brings the XML path into line. +if not opts.preserve_block_order: + P_list = [P_list[k] for k in np.random.permutation(len(P_list))] + lalsimutils.ChooseWaveformParams_array_to_xml(P_list,opts.output_file)