Container family: multi-architecture container deployment (port from rift_O4d)

MonteCarloMarginalizeCode/Code/bin/create_event_parameter_pipeline_BasicIteration

@@ -349,13 +349,35 @@ if opts.use_singularity:
     # SINGULARITY IMAGES ARE ON CVMFS, SO WE CAN AVOID THE SINGULARITY EXEC CALL
     # hardcoding a fiducial copy of lalapps_path2cache; beware about the executable name change
     os.environ['LALAPPS_PATH2CACHE'] = "/cvmfs/oasis.opensciencegrid.org/ligo/sw/conda/envs/igwn-py39/bin/lalapps_path2cache" #"singularity exec {singularity_image} lalapps_path2cache".format(singularity_image=singularity_image)
+    if os.environ.get('RIFT_CONTAINER_UNIVERSE') and ('SINGULARITY_BASE_EXE_DIR' in os.environ):
+        # Container universe deliberately does NOT set MY.SingularityBindCVMFS, so the
+        # hardcoded /cvmfs path above may not exist inside the image.  Use the copy the
+        # container itself ships (RIFT's own executables live in SINGULARITY_BASE_EXE_DIR).
+        os.environ['LALAPPS_PATH2CACHE'] = os.environ['SINGULARITY_BASE_EXE_DIR'].rstrip('/') + "/lal_path2cache"
     print(singularity_image)
 
     # see https://computing.docs.ligo.org/guide/htcondor/credentials/
     if 'igwn+osdf:' in singularity_image and not opts.use_oauth_files:
         opts.use_oauth_files = 'igwn' # force-set this variable, to save time for end user. This will always be the case
     elif 'osdf:' in singularity_image and not opts.use_oauth_files:
         opts.use_oauth_files = 'scitokens' # force-set this variable, to save time for end user. This will always be the case
+    elif not opts.use_oauth_files:
+        # Container FAMILY manifest (.yaml/.yml): the per-machine image osdf:// URLs live
+        # INSIDE the manifest, so the 'osdf:' substring checks above miss them and the
+        # transfer credential is never enabled -> the execute point cannot fetch the
+        # selected container ("credential is required for osdf://... but was not
+        # discovered" -> all ILE/CIP jobs held).  Inspect the manifest's image URLs and
+        # pick the same credential the single-image path would have.
+        try:
+            from RIFT.misc.container_manifest import is_container_manifest, load_container_manifest
+            if is_container_manifest(singularity_image):
+                _imgs = ' '.join(c.get('image', '') for c in load_container_manifest(singularity_image).get('containers', []))
+                if 'igwn+osdf:' in _imgs:
+                    opts.use_oauth_files = 'igwn'
+                elif 'osdf:' in _imgs:
+                    opts.use_oauth_files = 'scitokens'
+        except Exception:
+            pass
 
 if (opts.cip_args is None) and (opts.cip_args_list is None):
     print(" No arguments provided for low-level job")

MonteCarloMarginalizeCode/Code/bin/util_RIFT_pseudo_pipe.py

@@ -344,7 +344,17 @@ def unsafe_parse_arg_string_dict(my_argstr):
 
         if not('RIFT_REQUIRE_GPUS' in os.environ) and 'ile_require_gpus' in rift_items:
             os.environ['RIFT_REQUIRE_GPUS'] = rift_items['ile_require_gpus']
-
+
+        # Container family (multi-container per-machine image selection): let the ini
+        # file provide the image/exe-dir, as the environment normally would.  The value
+        # may be a single .sif/osdf URL (legacy) or a .yaml/.yml family manifest.
+        # Environment still dominates, matching the accounting/require-GPUs behavior above.
+        if not('SINGULARITY_RIFT_IMAGE' in os.environ) and 'singularity_rift_image' in rift_items:
+            os.environ['SINGULARITY_RIFT_IMAGE'] = rift_items['singularity_rift_image']
+        if not('SINGULARITY_BASE_EXE_DIR' in os.environ) and 'singularity_base_exe_dir' in rift_items:
+            os.environ['SINGULARITY_BASE_EXE_DIR'] = rift_items['singularity_base_exe_dir']
+
+
         # attempt to lazy-select the command-line that are present in the ini file section
         for item in rift_items:
             item_renamed = item.replace('-','_')