FIX TrueFalseResponseHandler: strip whitespace before validating verdict

pyrit/score/float_scale/plagiarism_scorer.py

@@ -124,6 +124,20 @@ def _ngram_set(self, tokens: list[str], n: int) -> set[tuple[str, ...]]:
         """
         return {tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)}
 
+    def _is_contiguous_sublist(self, sub: list[str], full: list[str]) -> bool:
+        """
+        Check whether ``sub`` appears as a contiguous run of tokens inside ``full``.
+
+        This mirrors the word-level tokenization the metrics rely on, so the
+        verbatim-match fast path stays consistent with them.
+
+        Returns:
+            bool: True if ``sub`` is a contiguous sublist of ``full``.
+        """
+        if not sub or len(sub) > len(full):
+            return False
+        return any(full[i : i + len(sub)] == sub for i in range(len(full) - len(sub) + 1))
+
     def _plagiarism_score(
         self,
         response: str,
@@ -139,8 +153,13 @@ def _plagiarism_score(
         if response_len == 0 or reference_len == 0:
             return 0.0
 
-        # If reference is in response, all three metrics should be 1.0
-        if reference in response:
+        # If the reference appears verbatim (word-level) in the response, all
+        # three metrics should be 1.0. Compare tokenized sequences rather than
+        # raw strings so this fast path matches the case/punctuation-insensitive
+        # tokenization used below, and so a short reference that is merely a
+        # substring of a longer response word (e.g. "cat" in "concatenate")
+        # does not falsely score as fully plagiarized.
+        if self._is_contiguous_sublist(tokens_reference, tokens_response):
             return 1.0
 
         # Compute the LCS metric (normalized by reference length)

pyrit/score/response_handler.py

@@ -303,7 +303,10 @@ def parse(
             objective=objective,
         )
 
-        normalized_value = score.raw_score_value.lower()
+        # Strip surrounding whitespace before comparing: a judge that returns
+        # "true\n" or " false" is giving a valid verdict, and should not be
+        # rejected as out-of-domain over incidental whitespace.
+        normalized_value = score.raw_score_value.strip().lower()
         if normalized_value not in {"true", "false"}:
             raise InvalidJsonException(
                 message=f"True/false score_value must be 'true' or 'false', not {score.raw_score_value!r}."

tests/unit/score/test_plagiarism_scorer.py

@@ -375,6 +375,34 @@ def test_plagiarism_score_jaccard_reference_contained_in_response(self, scorer):
         score = scorer._plagiarism_score(response, reference, metric=PlagiarismMetric.JACCARD, n=3)
         assert score == 1.0  # Should be perfect match when reference is contained
 
+    def test_plagiarism_score_reference_substring_of_word_not_plagiarism(self, scorer):
+        """A reference that is only a substring of a longer response word is not plagiarism.
+
+        The verbatim-match fast path must operate on word-level tokens, not raw
+        characters. Otherwise a short reference such as "cat" would falsely score
+        1.0 against a response containing "concatenate".
+        """
+        reference = "cat"
+        response = "concatenate the results"
+        for metric in PlagiarismMetric:
+            score = scorer._plagiarism_score(response, reference, metric=metric)
+            assert score == 0.0, f"{metric.value} should not treat a sub-word match as plagiarism"
+
+    def test_plagiarism_score_verbatim_match_ignores_case_and_punctuation(self, scorer):
+        """The verbatim fast path should still fire across case and punctuation differences."""
+        reference = "The Secret Plan"
+        response = "the secret plan!"
+        for metric in PlagiarismMetric:
+            score = scorer._plagiarism_score(response, reference, metric=metric)
+            assert score == 1.0, f"{metric.value} should treat a word-level verbatim copy as plagiarism"
+
+    def test_is_contiguous_sublist(self, scorer):
+        """Directly exercise the tokenized sublist helper."""
+        assert scorer._is_contiguous_sublist(["b", "c"], ["a", "b", "c", "d"]) is True
+        assert scorer._is_contiguous_sublist(["a", "c"], ["a", "b", "c"]) is False
+        assert scorer._is_contiguous_sublist([], ["a"]) is False
+        assert scorer._is_contiguous_sublist(["a", "b"], ["a"]) is False
+
 
 class TestPlagiarismMetricEnum:
     """Test cases for the PlagiarismMetric enum."""

tests/unit/score/test_response_handler.py

@@ -43,6 +43,29 @@ def test_true_false_response_handler_accepts_boolean_values(json_value: str, exp
     assert score.raw_score_value == expected
 
 
+@pytest.mark.parametrize(
+    ("json_value", "expected"),
+    [
+        ('"true "', "true"),
+        ('" false"', "false"),
+        ('"True\\n"', "true"),
+        ('"  FALSE  "', "false"),
+    ],
+)
+def test_true_false_response_handler_strips_whitespace_around_verdict(json_value: str, expected: str) -> None:
+    # A judge returning a valid verdict with incidental surrounding whitespace
+    # (e.g. a trailing newline) must not be rejected as out-of-domain.
+    handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())
+
+    score = handler.parse(
+        response_text=f'{{"score_value": {json_value}, "rationale": "test"}}',
+        scorer_identifier=SCORER_IDENTIFIER,
+        scored_prompt_id="test-id",
+    )
+
+    assert score.raw_score_value == expected
+
+
 def test_true_false_response_handler_rejects_value_outside_domain() -> None:
     handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())