Improve ONTAP benchmark protocol selection

Author: sp16743

private-cicd/benchmark/ontap/README.md

@@ -102,7 +102,8 @@ real lab IPs/credentials or run output.
 
 `run.py` is the entry point. Protocols, checkpoints, and concurrency levels
 come from `config.yaml` (`ontap` / `benchmark` for pools, `vm_bench` for
-VMs). Drop a protocol block to skip it, and edit `sequential_checkpoints` or
+VMs). Set `enabled_protocols` under `benchmark` or `vm_bench` to select
+`[nfs3]`, `[iscsi]`, or `[nfs3, iscsi]`, and edit `sequential_checkpoints` or
 `concurrency_levels` to change the matrix.
 
 Run ids are generated as `RUN_<UTC timestamp>_<4 hex chars>` (for example

private-cicd/benchmark/ontap/benchmark_vm_instance_concurrency.py

@@ -74,7 +74,6 @@
     load_config,
     make_cloudstack_client,
     RawLogger,
-    report_failed_creates,
     resolve_protocols,
 )
 
@@ -122,7 +121,6 @@ def run_concurrency(client, cfg, protocol_key, run_id, raw_logger, summary_rows,
         )
         for r in fail:
             log.error("%s: %s", r.vm_name, r.error)
-        report_failed_creates([(r.vm_name, r.vm_id) for r in fail])
 
         log.info("[6.2.2] Parallel DELETE protocol=%s C=%s", protocol_key, len(succ))
         t0 = time.perf_counter()

private-cicd/benchmark/ontap/benchmark_vm_instance_sequential.py

@@ -70,7 +70,6 @@
     load_config,
     make_cloudstack_client,
     RawLogger,
-    report_failed_creates,
     resolve_protocols,
 )
 
@@ -86,7 +85,6 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows,
     prefix = vm_cfg["vm_name_prefix"]
 
     created = []
-    failed = []
     create_durations = []
     log.info("[5.2.1] Sequential CREATE protocol=%s up to N=%s", protocol_key, max_n)
     for i in range(1, max_n + 1):
@@ -100,7 +98,6 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows,
             create_durations.append(result.duration_sec)
         else:
             log.error("%s", result.error)
-            failed.append((name, result.vm_id))
         if i in checkpoints:
             total = sum(create_durations)
             avg = mean_seconds(create_durations)
@@ -117,8 +114,6 @@ def run_sequential(client, cfg, protocol_key, run_id, raw_logger, summary_rows,
         if delay:
             time.sleep(delay)
 
-    report_failed_creates(failed)
-
     total_created = len(created)
     log.info("[5.2.2] Sequential DELETE protocol=%s from N=%s remaining", protocol_key, total_created)
     if total_created in checkpoints:

private-cicd/benchmark/ontap/config.example.yaml

@@ -76,6 +76,8 @@ ontap:
 
 benchmark:
   pool_name_prefix: "bench_ontap"  # ONTAP volume names disallow "-"; use "_" only
+  # Select any configured protocols without deleting their ontap blocks.
+  enabled_protocols: [nfs3, iscsi]
   # Checkpoints and concurrency levels are the combinations for a run.
   # Change them here.
   sequential_checkpoints: [1, 5, 10, 20, 30]
@@ -108,6 +110,8 @@ vm_bench:
   # Unattached data disks are deleted only from pools with this name prefix,
   # and only when that pool's provider and zone match this config.
   cleanup_pool_name_prefix: "bench_vm"
+  # Select any configured protocols without deleting their offering definitions.
+  enabled_protocols: [nfs3, iscsi]
   sequential_checkpoints: [1, 5, 10, 20]
   concurrency_levels: [2, 5, 10]
   inter_op_delay_sec: 0

private-cicd/benchmark/ontap/storage_pool_common.py

@@ -244,7 +244,16 @@ def load_config(path):
 def resolve_protocols(cfg, requested):
     available = list(cfg.get("ontap", {}).keys())
     if requested == "both":
-        return available
+        enabled = cfg.get("benchmark", {}).get("enabled_protocols", available)
+        unknown = [protocol for protocol in enabled if protocol not in available]
+        if unknown:
+            sys.exit(
+                f"Unknown config.benchmark.enabled_protocols entries: {unknown} "
+                f"(available: {available})"
+            )
+        if not enabled:
+            sys.exit("config.benchmark.enabled_protocols must contain at least one protocol")
+        return enabled
     if requested not in available:
         sys.exit(f"Protocol '{requested}' not found in config.ontap (available: {available})")
     return [requested]

private-cicd/benchmark/ontap/vm_instance_common.py

@@ -88,13 +88,9 @@ def build_deploy_vm_params(name, infra_cfg, vm_cfg, proto_cfg):
 
 def _extract_vm_id_from_error(error_text):
     """When deployVirtualMachine's async job fails deep in orchestration (for example
-    an ENOSPC-driven "Unable to orchestrate the start of VM instance" failure),
-    CloudStack has ALREADY created the VM (and its ROOT/DATA volume records) before
-    the failure - it just never finished starting it. The job's errortext embeds
-    that VM's uuid (e.g. '...{"instanceName":"i-2-107-VM", "uuid":"b8f4..."}.'),
-    so scrape it out here so the leftover VM/volume can at least be identified and
-    reported for manual inspection (see report_failed_creates()) instead of being
-    silently invisible."""
+    an "Unable to orchestrate the start of VM instance" failure), its errortext
+    can embed the attempted VM's UUID. Preserve that UUID in the raw operation
+    result even if CloudStack subsequently removes the failed VM."""
     if not error_text:
         return None
     m = _UUID_RE.search(error_text)
@@ -240,25 +236,6 @@ def append_summary_csv(path, rows):
         writer.writerows(rows)
 
 
-def report_failed_creates(failed):
-    """Failed deployVirtualMachine calls (for example ENOSPC "Unable to orchestrate
-    the start of VM instance" failures) still leave a real
-    VM + ROOT/DATA volume record behind in CloudStack - they just never finished
-    starting. These are intentionally NOT auto-destroyed here (unlike the
-    `created` list, which the benchmark itself deletes as part of 5.2.2/6.2.2) so
-    the leftover VM/volume artifacts stay inspectable afterwards - e.g. via
-    `listVolumes --state Destroy` on the bench_vm_<protocol> pool - to confirm
-    exactly what got left behind and why. Clean them up manually (or via
-    --cleanup-only) once you're done inspecting them."""
-    log = get_logger()
-    failed_with_id = [(name, vm_id) for name, vm_id in failed if vm_id]
-    if not failed_with_id:
-        return
-    log.info("%s failed-create VM(s) left in place for inspection (not cleaned up)", len(failed_with_id))
-    for name, vm_id in failed_with_id:
-        log.info("left in place: %s (%s)", name, vm_id)
-
-
 def _vm_name_pattern(prefix):
     return re.compile(
         rf"^{re.escape(prefix)}-(?:seq|c\d+)-[A-Za-z0-9]+-[A-Za-z0-9-]+-\d{{3}}$"
@@ -368,7 +345,16 @@ def load_config(path):
 def resolve_protocols(cfg, requested):
     available = list(cfg.get("vm_bench", {}).get("protocols", {}).keys())
     if requested == "both":
-        return available
+        enabled = cfg.get("vm_bench", {}).get("enabled_protocols", available)
+        unknown = [protocol for protocol in enabled if protocol not in available]
+        if unknown:
+            sys.exit(
+                f"Unknown config.vm_bench.enabled_protocols entries: {unknown} "
+                f"(available: {available})"
+            )
+        if not enabled:
+            sys.exit("config.vm_bench.enabled_protocols must contain at least one protocol")
+        return enabled
     if requested not in available:
         sys.exit(f"Protocol '{requested}' not found in config.vm_bench.protocols (available: {available})")
     return [requested]