diff --git a/src/google/adk/evaluation/agent_evaluator.py b/src/google/adk/evaluation/agent_evaluator.py index 3304b4359ae..67d6eef5ec0 100644 --- a/src/google/adk/evaluation/agent_evaluator.py +++ b/src/google/adk/evaluation/agent_evaluator.py @@ -60,6 +60,7 @@ from .eval_sets_manager import EvalSetsManager from .evaluator import EvalStatus from .in_memory_eval_sets_manager import InMemoryEvalSetsManager +from .llm_as_judge_utils import get_text_from_content from .local_eval_sets_manager import convert_eval_set_to_pydantic_schema from .simulation.user_simulator_provider import UserSimulatorProvider @@ -601,10 +602,7 @@ def _print_details( @staticmethod def _convert_content_to_text(content: Optional[genai_types.Content]) -> str: - if content and content.parts: - return "\n".join([p.text for p in content.parts if p.text]) - - return "" + return get_text_from_content(content) or "" @staticmethod def _convert_tool_calls_to_text( diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index 63559b26a9c..ccd0e218fab 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -32,6 +32,7 @@ from .evaluator import EvaluationResult from .evaluator import Evaluator from .evaluator import PerInvocationResult +from .llm_as_judge_utils import get_text_from_content class RougeEvaluator(Evaluator): @@ -90,10 +91,7 @@ def evaluate_invocations( def _get_text_from_content(content: Optional[genai_types.Content]) -> str: - if content and content.parts: - return "\n".join([part.text for part in content.parts if part.text]) - - return "" + return get_text_from_content(content) or "" def _get_eval_status(score: float, threshold: float) -> EvalStatus: diff --git a/src/google/adk/evaluation/hallucinations_v1.py b/src/google/adk/evaluation/hallucinations_v1.py index 5e32389316f..987d32e47d5 100644 --- a/src/google/adk/evaluation/hallucinations_v1.py +++ b/src/google/adk/evaluation/hallucinations_v1.py @@ -46,6 +46,7 @@ from .evaluator import PerInvocationResult from .llm_as_judge_utils import get_eval_status from .llm_as_judge_utils import get_text_from_content +from .llm_as_judge_utils import get_text_parts from .llm_as_judge_utils import get_tool_declarations_as_json_str logger = logging.getLogger("google_adk." + __name__) @@ -461,7 +462,7 @@ def _create_context_for_step( for part in event.content.parts if part.function_response ] - nl_responses = [part.text for part in event.content.parts if part.text] + nl_responses = get_text_parts(event.content) if nl_responses: context_parts.append("\n".join(nl_responses) + "\n") @@ -650,11 +651,7 @@ def _get_steps_to_evaluate(self, actual: Invocation) -> list[EvaluationStep]: if self._criterion.evaluate_intermediate_nl_responses: for event in all_events: - nl_parts = ( - [p.text for p in event.content.parts if p.text] - if event.content and event.content.parts - else [] - ) + nl_parts = get_text_parts(event.content) if nl_parts: context = self._create_context_for_step( actual.app_details, actual, events_for_context diff --git a/src/google/adk/evaluation/llm_as_judge_utils.py b/src/google/adk/evaluation/llm_as_judge_utils.py index 626cb9b3960..161673d11fe 100644 --- a/src/google/adk/evaluation/llm_as_judge_utils.py +++ b/src/google/adk/evaluation/llm_as_judge_utils.py @@ -47,6 +47,13 @@ class Label(enum.Enum): NOT_FOUND = "label field not found" +def get_text_parts(content: Optional[genai_types.Content]) -> list[str]: + """Returns the visible text parts of a `Content`, excluding thoughts.""" + if not content or not content.parts: + return [] + return [p.text for p in content.parts if p.text and not p.thought] + + def get_text_from_content( content: Optional[Union[genai_types.Content, Invocation]], *, @@ -87,7 +94,7 @@ def get_text_from_content( return "\n".join(parts) if parts else None if content and content.parts: - return "\n".join([p.text for p in content.parts if p.text]) + return "\n".join(get_text_parts(content)) return None diff --git a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py index 0fba74f79eb..bd7592efeed 100644 --- a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py +++ b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py @@ -30,6 +30,7 @@ from .evaluator import _validate_invocation_lengths from .evaluator import EvaluationResult from .evaluator import PerInvocationResult +from .llm_as_judge_utils import get_text_parts from .rubric_based_evaluator import RubricBasedEvaluator logger = logging.getLogger("google_adk." + __name__) @@ -193,7 +194,7 @@ def _assemble_dialogue_history( for turn_index, invocation in enumerate(actual_invocations): # USER TURN if invocation.user_content and invocation.user_content.parts: - text_parts = [p.text for p in invocation.user_content.parts if p.text] + text_parts = get_text_parts(invocation.user_content) if text_parts: dialogue_lines.append( f"USER TURN {turn_index + 1}: {' '.join(text_parts)}" @@ -208,7 +209,7 @@ def _assemble_dialogue_history( else f"AGENT ({event.author})" ) if event.content and event.content.parts: - text_parts = [p.text for p in event.content.parts if p.text] + text_parts = get_text_parts(event.content) if text_parts: dialogue_lines.append( f"{role} TURN {turn_index + 1}: {' '.join(text_parts)}" @@ -245,7 +246,7 @@ def _assemble_dialogue_history( ): agent_name = intermediate_data.invocation_events[0].author role = f"AGENT ({agent_name})" - text_parts = [p.text for p in invocation.final_response.parts if p.text] + text_parts = get_text_parts(invocation.final_response) if text_parts: dialogue_lines.append( f"{role} TURN {turn_index + 1}: {' '.join(text_parts)}" diff --git a/src/google/adk/evaluation/vertex_ai_eval_facade.py b/src/google/adk/evaluation/vertex_ai_eval_facade.py index 69d2b82f329..e418e12b67f 100644 --- a/src/google/adk/evaluation/vertex_ai_eval_facade.py +++ b/src/google/adk/evaluation/vertex_ai_eval_facade.py @@ -37,6 +37,7 @@ from .evaluator import EvaluationResult from .evaluator import Evaluator from .evaluator import PerInvocationResult +from .llm_as_judge_utils import get_text_from_content logger = logging.getLogger("google_adk." + __name__) @@ -113,10 +114,7 @@ def evaluate_invocations( """ def _get_text(self, content: Optional[genai_types.Content]) -> str: - if content and content.parts: - return "\n".join([p.text for p in content.parts if p.text]) - - return "" + return get_text_from_content(content) or "" def _get_score(self, eval_result: object) -> Optional[float]: summary_metrics: object = getattr(eval_result, "summary_metrics", None) diff --git a/tests/unittests/evaluation/test_agent_evaluator.py b/tests/unittests/evaluation/test_agent_evaluator.py index 3cc5eb256f8..6cfa6cf0eb6 100644 --- a/tests/unittests/evaluation/test_agent_evaluator.py +++ b/tests/unittests/evaluation/test_agent_evaluator.py @@ -488,6 +488,17 @@ def _make_result_with_invocation( ) +def test_convert_content_to_text_excludes_thought_parts(): + content = genai_types.Content( + parts=[ + genai_types.Part(text="Consider the options.", thought=True), + genai_types.Part(text="Paris"), + ] + ) + + assert AgentEvaluator._convert_content_to_text(content) == "Paris" + + def test_get_results_as_rows_flattens_metrics_and_invocations(): eval_metric_results = { "response_match_score": [ diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index cd7128f2aa3..c5bad7581b8 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -59,6 +59,62 @@ def _create_test_invocations( ) +@pytest.mark.parametrize( + "actual_parts, expected_parts, score", + [ + ( + [ + genai_types.Part( + text="Consider the capital of France.", thought=True + ), + genai_types.Part(text="Paris"), + ], + [genai_types.Part(text="Paris")], + 1.0, + ), + ( + [genai_types.Part(text="Paris", thought=False)], + [ + genai_types.Part( + text="Consider the capital of France.", thought=True + ), + genai_types.Part(text="Paris"), + ], + 1.0, + ), + ( + [genai_types.Part(text="Paris", thought=True)], + [genai_types.Part(text="Paris")], + 0.0, + ), + ( + [ + genai_types.Part(text="Paris", thought=True), + genai_types.Part(text="London"), + ], + [genai_types.Part(text="Paris")], + 0.0, + ), + ], +) +def test_response_match_scores_visible_text_only( + actual_parts, expected_parts, score +): + """Thought summaries neither dilute correct answers nor credit wrong ones.""" + actual, expected = _create_test_invocations("", "") + actual.final_response.parts = actual_parts + expected.final_response.parts = expected_parts + evaluator = _create_test_rouge_evaluator(threshold=0.5) + + result = evaluator.evaluate_invocations([actual], [expected]) + + assert result.overall_score == pytest.approx(score) + assert result.per_invocation_results[0].score == pytest.approx(score) + assert result.overall_eval_status == ( + EvalStatus.PASSED if score == 1.0 else EvalStatus.FAILED + ) + + def test_calculate_rouge_1_scores_empty_candidate_and_reference(): candidate = "" reference = "" diff --git a/tests/unittests/evaluation/test_final_response_match_v2.py b/tests/unittests/evaluation/test_final_response_match_v2.py index 2f6bc3b3385..3f865533d9c 100644 --- a/tests/unittests/evaluation/test_final_response_match_v2.py +++ b/tests/unittests/evaluation/test_final_response_match_v2.py @@ -268,6 +268,54 @@ def test_format_auto_rater_prompt_includes_intermediate_when_enabled(): assert "reference intro\nreference final" in prompt +def test_format_auto_rater_prompt_excludes_thought_parts(): + evaluator = _create_test_evaluator_gemini(threshold=0.8) + actual_invocation, expected_invocation = _create_test_invocations( + "candidate text", "reference text" + ) + actual_invocation.final_response.parts.insert( + 0, genai_types.Part(text="Considering the response.", thought=True) + ) + expected_invocation.final_response.parts.insert( + 0, genai_types.Part(text="Considering the reference.", thought=True) + ) + + prompt = evaluator.format_auto_rater_prompt( + actual_invocation, expected_invocation + ) + + assert "Considering the response." not in prompt + assert "Considering the reference." not in prompt + assert "candidate text" in prompt + assert "reference text" in prompt + + +def test_convert_auto_rater_response_to_score_ignores_judge_thought(): + """The judge model's own thought text must not break verdict parsing.""" + evaluator = _create_test_evaluator_gemini(threshold=0.8) + auto_rater_response = """```json +{ + "is_the_agent_response_valid": "valid", + "reasoning": "The response is valid." +} +```""" + llm_response = LlmResponse( + content=genai_types.Content( + parts=[ + genai_types.Part( + text="Let me evaluate this response.", thought=True + ), + genai_types.Part(text=auto_rater_response), + ], + role="model", + ) + ) + auto_rater_score = evaluator.convert_auto_rater_response_to_score( + llm_response + ) + assert auto_rater_score == AutoRaterScore(score=1.0) + + def test_convert_auto_rater_response_to_score_valid(): evaluator = _create_test_evaluator_gemini(threshold=0.8) auto_rater_response = """```json diff --git a/tests/unittests/evaluation/test_hallucinations_v1.py b/tests/unittests/evaluation/test_hallucinations_v1.py index b7a1c42f47d..e19b76c5bbb 100644 --- a/tests/unittests/evaluation/test_hallucinations_v1.py +++ b/tests/unittests/evaluation/test_hallucinations_v1.py @@ -577,6 +577,116 @@ def agent_tree_data(): return invocation, expected_invocation +class TestCreateContextExcludesThoughts: + """Test cases ensuring thought parts do not leak into the context.""" + + def test_create_context_excludes_thought_from_nl_response( + self, hallucinations_metric + ): + """A thought part must not appear in the assembled context string.""" + app_details = AppDetails( + agent_details={ + "root": AgentDetails( + name="root", instructions="Root agent instructions." + ) + }, + ) + user_content = genai_types.Content( + parts=[genai_types.Part(text="User query.")] + ) + events = [ + InvocationEvent( + author="root", + content=genai_types.Content( + parts=[ + genai_types.Part( + text="Considering the query.", thought=True + ), + genai_types.Part(text="Visible NL response."), + ] + ), + ), + ] + invocation = Invocation( + app_details=app_details, + user_content=user_content, + intermediate_data=InvocationEvents(invocation_events=events), + ) + + context = hallucinations_metric._create_context_for_step( + app_details, invocation, events + ) + + assert "Considering the query." not in context + assert "Visible NL response." in context + + +class TestGetStepsToEvaluateExcludesThoughts: + """Test cases ensuring thought parts do not become their own eval steps.""" + + def test_thought_only_event_yields_no_step(self, hallucinations_metric): + """A thought-only intermediate event must not produce an evaluation step.""" + app_details = AppDetails(agent_details={}) + user_content = genai_types.Content( + parts=[genai_types.Part(text="User query.")] + ) + events = [ + InvocationEvent( + author="root", + content=genai_types.Content( + parts=[genai_types.Part(text="Just thinking.", thought=True)] + ), + ), + ] + invocation = Invocation( + app_details=app_details, + user_content=user_content, + intermediate_data=InvocationEvents(invocation_events=events), + final_response=genai_types.Content( + parts=[genai_types.Part(text="Final response.")] + ), + ) + + steps = hallucinations_metric._get_steps_to_evaluate(invocation) + + assert [step.nl_response for step in steps] == ["Final response."] + + def test_mixed_event_does_not_split_thought_into_its_own_step( + self, hallucinations_metric + ): + """A thought alongside visible text yields one step, not two.""" + app_details = AppDetails(agent_details={}) + user_content = genai_types.Content( + parts=[genai_types.Part(text="User query.")] + ) + events = [ + InvocationEvent( + author="root", + content=genai_types.Content( + parts=[ + genai_types.Part(text="Reasoning aloud.", thought=True), + genai_types.Part(text="Visible NL response."), + ] + ), + ), + ] + invocation = Invocation( + app_details=app_details, + user_content=user_content, + intermediate_data=InvocationEvents(invocation_events=events), + final_response=genai_types.Content( + parts=[genai_types.Part(text="Final response.")] + ), + ) + + steps = hallucinations_metric._get_steps_to_evaluate(invocation) + + assert [step.nl_response for step in steps] == [ + "Visible NL response.", + "Final response.", + ] + + class TestEvaluateInvocationsAgentTree: """Test cases for agent tree.""" diff --git a/tests/unittests/evaluation/test_llm_as_judge_utils.py b/tests/unittests/evaluation/test_llm_as_judge_utils.py index 6e6dd8772a4..f09a2692ca2 100644 --- a/tests/unittests/evaluation/test_llm_as_judge_utils.py +++ b/tests/unittests/evaluation/test_llm_as_judge_utils.py @@ -28,6 +28,7 @@ from google.adk.evaluation.llm_as_judge_utils import get_eval_status from google.adk.evaluation.llm_as_judge_utils import get_grounding_metadata_as_json_str from google.adk.evaluation.llm_as_judge_utils import get_text_from_content +from google.adk.evaluation.llm_as_judge_utils import get_text_parts from google.adk.evaluation.llm_as_judge_utils import get_tool_calls_and_responses_as_json_str from google.adk.evaluation.llm_as_judge_utils import get_tool_declarations_as_json_str from google.genai import types as genai_types @@ -133,6 +134,82 @@ def test_get_text_from_content_with_invocation_include_intermediate_responses_in ) +def test_get_text_from_content_excludes_thought_parts(): + """Tests get_text_from_content excludes parts marked as thoughts.""" + content = genai_types.Content( + parts=[ + genai_types.Part(text="Let me think about this.", thought=True), + genai_types.Part(text="Paris"), + ] + ) + assert get_text_from_content(content) == "Paris" + + +def test_get_text_from_content_with_only_thought_parts(): + """Tests get_text_from_content returns empty string for thought-only content.""" + content = genai_types.Content( + parts=[genai_types.Part(text="Just thinking.", thought=True)] + ) + assert get_text_from_content(content) == "" + + +def test_get_text_from_content_excludes_thoughts_from_intermediate_events(): + """Tests thought parts are excluded from intermediate responses too.""" + invocation = Invocation( + user_content=genai_types.Content(parts=[genai_types.Part(text="user")]), + intermediate_data=InvocationEvents( + invocation_events=[ + InvocationEvent( + author="agent", + content=genai_types.Content( + parts=[ + genai_types.Part( + text="Considering options.", thought=True + ), + genai_types.Part(text="Let me check."), + ] + ), + ), + ] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="Done.")] + ), + ) + + assert ( + get_text_from_content( + invocation, include_intermediate_responses_in_final=True + ) + == "Let me check.\nDone." + ) + + +def test_get_text_parts_with_none(): + """Tests get_text_parts returns an empty list for None.""" + assert get_text_parts(None) == [] + + +def test_get_text_parts_with_no_parts(): + """Tests get_text_parts returns an empty list when parts is None.""" + assert get_text_parts(genai_types.Content(parts=None)) == [] + + +def test_get_text_parts_excludes_thoughts(): + """Tests get_text_parts excludes thought parts and non-text parts.""" + content = genai_types.Content( + parts=[ + genai_types.Part(text="Thinking...", thought=True), + genai_types.Part(text="Hello"), + genai_types.Part( + function_call=genai_types.FunctionCall(name="test_func") + ), + genai_types.Part(text="World"), + ] + ) + assert get_text_parts(content) == ["Hello", "World"] + + def test_get_text_from_content_with_intermediate_data_full_response(): invocation = Invocation( user_content=genai_types.Content(parts=[genai_types.Part(text="user")]), diff --git a/tests/unittests/evaluation/test_rubric_based_final_response_quality_v1.py b/tests/unittests/evaluation/test_rubric_based_final_response_quality_v1.py index 27ed2d7ed17..9e398dc7105 100644 --- a/tests/unittests/evaluation/test_rubric_based_final_response_quality_v1.py +++ b/tests/unittests/evaluation/test_rubric_based_final_response_quality_v1.py @@ -112,6 +112,29 @@ def test_format_auto_rater_prompt_with_basic_invocation( ) in prompt +def test_format_auto_rater_prompt_excludes_thought_parts( + evaluator: RubricBasedFinalResponseQualityV1Evaluator, +): + """Tests format_auto_rater_prompt excludes thought parts from the response.""" + invocation = Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="User input here.")] + ), + final_response=genai_types.Content( + parts=[ + genai_types.Part( + text="Considering how to respond.", thought=True + ), + genai_types.Part(text="Final agent response."), + ] + ), + ) + prompt = evaluator.format_auto_rater_prompt(invocation, None) + + assert "Considering how to respond." not in prompt + assert "Final agent response." in prompt + + def test_format_auto_rater_prompt_with_app_details( evaluator: RubricBasedFinalResponseQualityV1Evaluator, ): diff --git a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py index e926d0b7203..ed94a18c589 100644 --- a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py @@ -285,6 +285,86 @@ async def test_app_details_instructions_and_tools(self, evaluator): assert "transfer_funds" in evaluator._formatted_tools assert "Transfer money between accounts." in evaluator._formatted_tools + @pytest.mark.asyncio + async def test_user_turn_excludes_thought_parts(self, evaluator): + """Tests that a thought part in the user turn is excluded from dialogue.""" + invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[ + genai_types.Part(text="Considering...", thought=True), + genai_types.Part(text="Hello"), + ] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="Hi there!")] + ), + invocation_id="agent1", + rubrics=_RUBRICS, + ), + ] + evaluator._assemble_dialogue_history(invocations) + + assert "Considering..." not in evaluator._formatted_dialogue + assert "USER TURN 1: Hello" in evaluator._formatted_dialogue + + @pytest.mark.asyncio + async def test_intermediate_event_excludes_thought_parts(self, evaluator): + """Tests that a thought part in an intermediate event is excluded.""" + intermediate_data = InvocationEvents( + invocation_events=[ + InvocationEvent( + author="banking_agent", + content=genai_types.Content( + parts=[ + genai_types.Part( + text="Let me check the balance.", thought=True + ), + genai_types.Part(text="Checking your balance."), + ] + ), + ), + ] + ) + invocations = [ + _make_invocation( + user_text="What is my balance?", + agent_text="Your balance is $100.", + invocation_id="banking_agent", + rubrics=_RUBRICS, + intermediate_data=intermediate_data, + ), + ] + evaluator._assemble_dialogue_history(invocations) + + assert "Let me check the balance." not in evaluator._formatted_dialogue + assert "Checking your balance." in evaluator._formatted_dialogue + + @pytest.mark.asyncio + async def test_final_response_excludes_thought_parts(self, evaluator): + """Tests that a thought part in the final response is excluded.""" + invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="Hello")] + ), + final_response=genai_types.Content( + parts=[ + genai_types.Part( + text="Deciding how to respond.", thought=True + ), + genai_types.Part(text="Hi there!"), + ] + ), + invocation_id="agent1", + rubrics=_RUBRICS, + ), + ] + evaluator._assemble_dialogue_history(invocations) + + assert "Deciding how to respond." not in evaluator._formatted_dialogue + assert "AGENT (agent) TURN 1: Hi there!" in evaluator._formatted_dialogue + @pytest.mark.asyncio async def test_invocation_without_user_content(self, evaluator): """Tests that invocations with no user text parts are handled gracefully.""" diff --git a/tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py b/tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py index 64d5ff7a313..b340a7bcb73 100644 --- a/tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py +++ b/tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py @@ -79,6 +79,24 @@ def test_format_auto_rater_prompt_with_basic_invocation( assert "\nNo intermediate steps were taken.\n" in prompt +def test_format_auto_rater_prompt_excludes_thought_parts( + evaluator: RubricBasedToolUseV1Evaluator, +): + """Tests format_auto_rater_prompt excludes thought parts from user input.""" + invocation = Invocation( + user_content=genai_types.Content( + parts=[ + genai_types.Part(text="Considering the request.", thought=True), + genai_types.Part(text="User input here."), + ] + ), + ) + prompt = evaluator.format_auto_rater_prompt(invocation, None) + + assert "Considering the request." not in prompt + assert "User input here." in prompt + + def test_format_auto_rater_prompt_with_invocation_rubrics_only(): """Tests prompt formatting when rubrics are defined on the invocation.""" judge_model_options = JudgeModelOptions( diff --git a/tests/unittests/evaluation/test_vertex_ai_eval_facade.py b/tests/unittests/evaluation/test_vertex_ai_eval_facade.py index 285f8c70629..11df6c57cc8 100644 --- a/tests/unittests/evaluation/test_vertex_ai_eval_facade.py +++ b/tests/unittests/evaluation/test_vertex_ai_eval_facade.py @@ -36,6 +36,22 @@ vertexai_types = vertexai.types +def test_get_text_excludes_thought_parts(mocker): + """Tests _get_text excludes thought parts from the extracted text.""" + mocker.patch("google.adk.dependencies.vertexai.vertexai.Client") + evaluator = _SingleTurnVertexAiEvalFacade( + threshold=0.8, metric_name=vertexai_types.PrebuiltMetric.COHERENCE + ) + content = genai_types.Content( + parts=[ + genai_types.Part(text="Consider the options.", thought=True), + genai_types.Part(text="Paris"), + ] + ) + + assert evaluator._get_text(content) == "Paris" + + class TestSingleTurnVertexAiEvalFacade: """A class to help organize "patch" that are applicable to all tests."""