[Evaluation] Configure adaptive native AOAI output result page size

.chronus/changes/rboubchir-azure-ai-evaluation-aoai-output-page-size-2026-09-15.md

@@ -0,0 +1,9 @@
+---
+changeKind: feature
+packages:
+  - azure-ai-evaluation
+---
+
+Added the `aoai_output_items_page_size` option to `evaluate` for configuring the number of native Azure OpenAI
+grader output items requested per HTTP response page. The default remains 100, all result pages are fetched, and
+timed-out output-item requests retry the same page with progressively smaller page sizes.

sdk/evaluation/azure-ai-evaluation/api.md

@@ -3,6 +3,7 @@ namespace azure.ai.evaluation
 
     def azure.ai.evaluation.evaluate(
             *, 
+            aoai_output_items_page_size: int = 100,
             azure_ai_project: Optional[Union[str, AzureAIProject]] = ..., 
             data: Union[str, PathLike], 
             evaluation_name: Optional[str] = ..., 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -59,6 +59,8 @@
 from ._batch_run.batch_clients import BatchClient, BatchClientRun
 
 from ._evaluate_aoai import (
+    _DEFAULT_AOAI_OUTPUT_ITEMS_PAGE_SIZE,
+    _MAX_AOAI_OUTPUT_ITEMS_PAGE_SIZE,
     _begin_aoai_evaluation,
     _split_evaluators_and_grader_configs,
     _get_evaluation_run_results,
@@ -818,6 +820,25 @@ def _rename_columns_conditionally(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def _validate_aoai_output_items_page_size(aoai_output_items_page_size: int) -> None:
+    if (
+        not isinstance(aoai_output_items_page_size, int)
+        or isinstance(aoai_output_items_page_size, bool)
+        or not 1 <= aoai_output_items_page_size <= _MAX_AOAI_OUTPUT_ITEMS_PAGE_SIZE
+    ):
+        msg = (
+            "'aoai_output_items_page_size' must be an integer between 1 and "
+            f"{_MAX_AOAI_OUTPUT_ITEMS_PAGE_SIZE}, inclusive."
+        )
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            target=ErrorTarget.EVALUATE,
+            category=ErrorCategory.INVALID_VALUE,
+            blame=ErrorBlame.USER_ERROR,
+        )
+
+
 def evaluate(
     *,
     data: Union[str, os.PathLike],
@@ -828,6 +849,7 @@ def evaluate(
     azure_ai_project: Optional[Union[str, AzureAIProject]] = None,
     output_path: Optional[Union[str, os.PathLike]] = None,
     fail_on_evaluator_errors: bool = False,
+    aoai_output_items_page_size: int = 100,
     tags: Optional[Dict[str, str]] = None,
     **kwargs,
 ) -> EvaluationResult:
@@ -861,6 +883,11 @@ def evaluate(
         Defaults to false, which means that evaluations will continue regardless of failures.
         If such failures occur, metrics may be missing, and evidence of failures can be found in the evaluation's logs.
     :paramtype fail_on_evaluator_errors: bool
+    :keyword aoai_output_items_page_size: The maximum number of native Azure OpenAI grader output items requested
+        per HTTP response page. Defaults to 100. This does not limit response bytes, request latency, or the number
+        of dataset rows evaluated; all output-item result pages are fetched. If an output-items request times out or
+        returns HTTP 408 or 504, the same page is retried with a smaller page size.
+    :paramtype aoai_output_items_page_size: int
     :keyword tags: A dictionary of tags to be added to the evaluation run for tracking and organization purposes.
         Keys and values must be strings. For more information about tag limits, see:
         https://learn.microsoft.com/en-us/azure/machine-learning/resource-limits-capacity?view=azureml-api-2#runs
@@ -890,6 +917,7 @@ def evaluate(
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
     """
     try:
+        _validate_aoai_output_items_page_size(aoai_output_items_page_size)
         user_agent: Optional[str] = kwargs.get("user_agent")
         with UserAgentSingleton().add_useragent_product(user_agent) if user_agent else contextlib.nullcontext():
             results = _evaluate(
@@ -901,6 +929,7 @@ def evaluate(
                 azure_ai_project=azure_ai_project,
                 output_path=output_path,
                 fail_on_evaluator_errors=fail_on_evaluator_errors,
+                aoai_output_items_page_size=aoai_output_items_page_size,
                 tags=tags,
                 **kwargs,
             )
@@ -971,6 +1000,7 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
     azure_ai_project: Optional[Union[str, AzureAIProject]] = None,
     output_path: Optional[Union[str, os.PathLike]] = None,
     fail_on_evaluator_errors: bool = False,
+    aoai_output_items_page_size: int = _DEFAULT_AOAI_OUTPUT_ITEMS_PAGE_SIZE,
     tags: Optional[Dict[str, str]] = None,
     **kwargs,
 ) -> EvaluationResult:
@@ -1048,7 +1078,9 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
     # Retrieve OAI eval run results if needed.
     if need_get_oai_results:
         try:
-            aoai_results, aoai_metrics = _get_evaluation_run_results(eval_run_info_list)  # type: ignore
+            aoai_results, aoai_metrics = _get_evaluation_run_results(
+                eval_run_info_list, aoai_output_items_page_size
+            )  # type: ignore
             # Post build TODO: add equivalent of  _print_summary(per_evaluator_results) here
 
             # Combine results if both evaluators and graders are present

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py

@@ -5,23 +5,26 @@
 import json
 import logging
 import re
-
-from openai import AzureOpenAI, OpenAI
-import pandas as pd
-from typing import Any, Callable, Dict, Tuple, TypeVar, Union, Type, Optional, TypedDict, List, cast, Set
 from time import sleep
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypedDict, TypeVar, Type, Union, cast
 
-from ._batch_run import CodeClient, ProxyClient
+import pandas as pd
+from openai import APIConnectionError, APIStatusError, APITimeoutError, AzureOpenAI, OpenAI
+from openai._models import FinalRequestOptions
 
 # import aoai_mapping
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
 from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
+from ._batch_run import CodeClient, ProxyClient
 
 TClient = TypeVar("TClient", ProxyClient, CodeClient)
 LOGGER = logging.getLogger(__name__)
+_DEFAULT_AOAI_OUTPUT_ITEMS_PAGE_SIZE = 100
+_MAX_AOAI_OUTPUT_ITEMS_PAGE_SIZE = 100
+_AOAI_OUTPUT_ITEMS_MAX_ATTEMPTS = 3
 
 # Precompiled regex for extracting data paths from mapping expressions of the form
 # ${data.some.dotted.path}. Compiled once at import time to avoid repeated
@@ -258,7 +261,10 @@ def _combine_item_schemas(data_source_config: Dict[str, Any], kwargs: Dict[str,
                     data_source_config["item_schema"]["required"].append(key)
 
 
-def _get_evaluation_run_results(all_run_info: List[OAIEvalRunCreationInfo]) -> Tuple[pd.DataFrame, Dict[str, Any]]:
+def _get_evaluation_run_results(
+    all_run_info: List[OAIEvalRunCreationInfo],
+    aoai_output_items_page_size: int = _DEFAULT_AOAI_OUTPUT_ITEMS_PAGE_SIZE,
+) -> Tuple[pd.DataFrame, Dict[str, Any]]:
     """
     Get the results of an OAI evaluation run, formatted in a way that is easy for the rest of the evaluation
     pipeline to consume. This method accepts a list of eval run information, and will combine the
@@ -267,6 +273,8 @@ def _get_evaluation_run_results(all_run_info: List[OAIEvalRunCreationInfo]) -> T
     :param all_run_info: A list of evaluation run information that contains the needed values
         to retrieve the results of the evaluation run.
     :type all_run_info: List[OAIEvalRunCreationInfo]
+    :param aoai_output_items_page_size: The maximum number of output items to request per page.
+    :type aoai_output_items_page_size: int
     :return: A tuple containing the results of the evaluation run as a dataframe, and a dictionary of metrics
         calculated from the evaluation run.
     :rtype: Tuple[pd.DataFrame, Dict[str, Any]]
@@ -278,16 +286,75 @@ def _get_evaluation_run_results(all_run_info: List[OAIEvalRunCreationInfo]) -> T
     output_df = pd.DataFrame()
     for idx, run_info in enumerate(all_run_info):
         LOGGER.info(f"AOAI: Fetching results for run {idx + 1}/{len(all_run_info)} (ID: {run_info['eval_run_id']})...")
-        cur_output_df, cur_run_metrics = _get_single_run_results(run_info)
+        cur_output_df, cur_run_metrics = _get_single_run_results(run_info, aoai_output_items_page_size)
         output_df = pd.concat([output_df, cur_output_df], axis=1)
         run_metrics.update(cur_run_metrics)
 
     LOGGER.info(f"AOAI: Successfully retrieved all results. Combined dataframe shape: {output_df.shape}")
     return output_df, run_metrics
 
 
+def _list_output_items_page(
+    client: Union[AzureOpenAI, OpenAI],
+    list_kwargs: Dict[str, Any],
+    page_size: int,
+) -> Tuple[Any, int]:
+    """Fetch one output-items page with the OpenAI client's retry policy and a three-attempt budget.
+
+    :param client: A scoped OpenAI client with automatic retries disabled.
+    :type client: Union[AzureOpenAI, OpenAI]
+    :param list_kwargs: Arguments identifying the evaluation run and current cursor.
+    :type list_kwargs: Dict[str, Any]
+    :param page_size: The number of output items to request.
+    :type page_size: int
+    :return: The fetched page and the page size to retain for subsequent pages.
+    :rtype: Tuple[Any, int]
+    """
+    retry_options = FinalRequestOptions(
+        method="get",
+        url="/evals/runs/output_items",
+        max_retries=_AOAI_OUTPUT_ITEMS_MAX_ATTEMPTS - 1,
+    )
+
+    for attempt in range(_AOAI_OUTPUT_ITEMS_MAX_ATTEMPTS):
+        try:
+            return client.evals.runs.output_items.list(**list_kwargs, limit=page_size), page_size
+        except (APIConnectionError, APIStatusError) as error:
+            should_reduce_page_size = isinstance(error, APITimeoutError)
+            should_retry = isinstance(error, APIConnectionError)
+            response_headers = None
+
+            if isinstance(error, APIStatusError):
+                should_retry = client._should_retry(error.response)  # pylint: disable=protected-access
+                should_reduce_page_size = should_retry and error.status_code in (408, 504)
+                response_headers = error.response.headers
+
+            if not should_retry or attempt == _AOAI_OUTPUT_ITEMS_MAX_ATTEMPTS - 1:
+                raise
+
+            if should_reduce_page_size:
+                page_size = max(1, (page_size + 1) // 2)
+
+            remaining_retries = _AOAI_OUTPUT_ITEMS_MAX_ATTEMPTS - attempt - 1
+            delay = client._calculate_retry_timeout(  # pylint: disable=protected-access
+                remaining_retries,
+                retry_options,
+                response_headers,
+            )
+            LOGGER.warning(
+                "AOAI output-items request failed for cursor %s. Retrying with page size %d in %.2f seconds.",
+                list_kwargs.get("after"),
+                page_size,
+                delay,
+            )
+            sleep(delay)
+
+    raise RuntimeError("AOAI output-items retry loop exited unexpectedly.")
+
+
 def _get_single_run_results(
     run_info: OAIEvalRunCreationInfo,
+    aoai_output_items_page_size: int = _DEFAULT_AOAI_OUTPUT_ITEMS_PAGE_SIZE,
 ) -> Tuple[pd.DataFrame, Dict[str, Any]]:
     """
     Get the results of an OAI evaluation run, formatted in a way that is easy for the rest of the evaluation
@@ -296,6 +363,8 @@ def _get_single_run_results(
     :param run_info: The evaluation run information that contains the needed values
         to retrieve the results of the evaluation run.
     :type run_info: OAIEvalRunCreationInfo
+    :param aoai_output_items_page_size: The maximum number of output items to request per page.
+    :type aoai_output_items_page_size: int
     :return: A tuple containing the results of the evaluation run as a dataframe, and a dictionary of metrics
         calculated from the evaluation run.
     :rtype: Tuple[pd.DataFrame, Dict[str, Any]]
@@ -349,14 +418,18 @@ def _get_single_run_results(
     LOGGER.info(f"AOAI: Collecting output items for run {run_info['eval_run_id']} with pagination...")
     all_results: List[Any] = []
     next_cursor: Optional[str] = None
-    limit = 100  # Max allowed by API
+    page_size = aoai_output_items_page_size
+    output_items_client = run_info["client"].with_options(max_retries=0)
 
     while True:
-        list_kwargs = {"eval_id": run_info["eval_group_id"], "run_id": run_info["eval_run_id"], "limit": limit}
+        list_kwargs = {
+            "eval_id": run_info["eval_group_id"],
+            "run_id": run_info["eval_run_id"],
+        }
         if next_cursor is not None:
             list_kwargs["after"] = next_cursor
 
-        raw_list_results = run_info["client"].evals.runs.output_items.list(**list_kwargs)
+        raw_list_results, page_size = _list_output_items_page(output_items_client, list_kwargs, page_size)
 
         # Add current page results
         all_results.extend(raw_list_results.data)

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_aoai_alignment_missing_rows.py

@@ -32,6 +32,7 @@ def __init__(self, data, has_more=False):
 def test_aoai_results_preserve_order_with_unordered_output_items(caplog):
     """AOAI output_items can arrive unordered; results should align to row ids (0..N-1)."""
     mock_client = Mock()
+    mock_client.with_options.return_value = mock_client
     expected_rows = 5
     run_info = OAIEvalRunCreationInfo(
         client=mock_client,