From 0276f579d05acdbb7eef52563e3f954256469bfa Mon Sep 17 00:00:00 2001 From: WatchTree-19 <119982314+WatchTree-19@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:43:26 +0100 Subject: [PATCH 1/2] fix(scorer): PlagiarismScorer verbatim fast path matched sub-word substrings The verbatim-match shortcut in _plagiarism_score used a raw string check (reference in response). The rest of the scorer is word-level: it tokenizes with lowercasing and punctuation removal before computing LCS / Levenshtein / Jaccard. The raw check was inconsistent with that in both directions: - false positive: a short reference that is only a substring of a longer response word scored 1.0 (e.g. reference 'cat' vs response 'concatenate the results' returned full plagiarism for every metric). - missed match: a word-level verbatim copy differing only in case or punctuation did not take the fast path. Compare the tokenized sequences instead, so the fast path matches the same word-level semantics the metrics use. Adds regression tests. --- pyrit/score/float_scale/plagiarism_scorer.py | 23 ++++++++++++++-- tests/unit/score/test_plagiarism_scorer.py | 28 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index b579418bdc..3e78a2daf3 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -124,6 +124,20 @@ def _ngram_set(self, tokens: list[str], n: int) -> set[tuple[str, ...]]: """ return {tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)} + def _is_contiguous_sublist(self, sub: list[str], full: list[str]) -> bool: + """ + Check whether ``sub`` appears as a contiguous run of tokens inside ``full``. + + This mirrors the word-level tokenization the metrics rely on, so the + verbatim-match fast path stays consistent with them. + + Returns: + bool: True if ``sub`` is a contiguous sublist of ``full``. + """ + if not sub or len(sub) > len(full): + return False + return any(full[i : i + len(sub)] == sub for i in range(len(full) - len(sub) + 1)) + def _plagiarism_score( self, response: str, @@ -139,8 +153,13 @@ def _plagiarism_score( if response_len == 0 or reference_len == 0: return 0.0 - # If reference is in response, all three metrics should be 1.0 - if reference in response: + # If the reference appears verbatim (word-level) in the response, all + # three metrics should be 1.0. Compare tokenized sequences rather than + # raw strings so this fast path matches the case/punctuation-insensitive + # tokenization used below, and so a short reference that is merely a + # substring of a longer response word (e.g. "cat" in "concatenate") + # does not falsely score as fully plagiarized. + if self._is_contiguous_sublist(tokens_reference, tokens_response): return 1.0 # Compute the LCS metric (normalized by reference length) diff --git a/tests/unit/score/test_plagiarism_scorer.py b/tests/unit/score/test_plagiarism_scorer.py index aef799eea6..78826a3c37 100644 --- a/tests/unit/score/test_plagiarism_scorer.py +++ b/tests/unit/score/test_plagiarism_scorer.py @@ -375,6 +375,34 @@ def test_plagiarism_score_jaccard_reference_contained_in_response(self, scorer): score = scorer._plagiarism_score(response, reference, metric=PlagiarismMetric.JACCARD, n=3) assert score == 1.0 # Should be perfect match when reference is contained + def test_plagiarism_score_reference_substring_of_word_not_plagiarism(self, scorer): + """A reference that is only a substring of a longer response word is not plagiarism. + + The verbatim-match fast path must operate on word-level tokens, not raw + characters. Otherwise a short reference such as "cat" would falsely score + 1.0 against a response containing "concatenate". + """ + reference = "cat" + response = "concatenate the results" + for metric in PlagiarismMetric: + score = scorer._plagiarism_score(response, reference, metric=metric) + assert score == 0.0, f"{metric.value} should not treat a sub-word match as plagiarism" + + def test_plagiarism_score_verbatim_match_ignores_case_and_punctuation(self, scorer): + """The verbatim fast path should still fire across case and punctuation differences.""" + reference = "The Secret Plan" + response = "the secret plan!" + for metric in PlagiarismMetric: + score = scorer._plagiarism_score(response, reference, metric=metric) + assert score == 1.0, f"{metric.value} should treat a word-level verbatim copy as plagiarism" + + def test_is_contiguous_sublist(self, scorer): + """Directly exercise the tokenized sublist helper.""" + assert scorer._is_contiguous_sublist(["b", "c"], ["a", "b", "c", "d"]) is True + assert scorer._is_contiguous_sublist(["a", "c"], ["a", "b", "c"]) is False + assert scorer._is_contiguous_sublist([], ["a"]) is False + assert scorer._is_contiguous_sublist(["a", "b"], ["a"]) is False + class TestPlagiarismMetricEnum: """Test cases for the PlagiarismMetric enum.""" From 3759fed7a17d40d172dcbf49ff4232412e572501 Mon Sep 17 00:00:00 2001 From: WatchTree-19 <119982314+WatchTree-19@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:46:40 +0100 Subject: [PATCH 2/2] FIX TrueFalseResponseHandler: strip whitespace before validating verdict The true/false response handler lowercased the parsed score value but did not strip it before checking membership in {"true", "false"}. A judge returning a valid verdict with incidental surrounding whitespace - e.g. 'true\n', ' false', or 'True ' from a target that does not enforce the JSON schema - was rejected as out-of-domain, discarding a usable score (and triggering the JSON retry path). Strip before lowercasing so incidental whitespace no longer invalidates an otherwise-valid true/false verdict. Adds a parametrized regression test. --- pyrit/score/response_handler.py | 5 ++++- tests/unit/score/test_response_handler.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pyrit/score/response_handler.py b/pyrit/score/response_handler.py index 6dea0a9d0a..0c27efd120 100644 --- a/pyrit/score/response_handler.py +++ b/pyrit/score/response_handler.py @@ -303,7 +303,10 @@ def parse( objective=objective, ) - normalized_value = score.raw_score_value.lower() + # Strip surrounding whitespace before comparing: a judge that returns + # "true\n" or " false" is giving a valid verdict, and should not be + # rejected as out-of-domain over incidental whitespace. + normalized_value = score.raw_score_value.strip().lower() if normalized_value not in {"true", "false"}: raise InvalidJsonException( message=f"True/false score_value must be 'true' or 'false', not {score.raw_score_value!r}." diff --git a/tests/unit/score/test_response_handler.py b/tests/unit/score/test_response_handler.py index 16084a5cf6..fac1b55ac0 100644 --- a/tests/unit/score/test_response_handler.py +++ b/tests/unit/score/test_response_handler.py @@ -43,6 +43,29 @@ def test_true_false_response_handler_accepts_boolean_values(json_value: str, exp assert score.raw_score_value == expected +@pytest.mark.parametrize( + ("json_value", "expected"), + [ + ('"true "', "true"), + ('" false"', "false"), + ('"True\\n"', "true"), + ('" FALSE "', "false"), + ], +) +def test_true_false_response_handler_strips_whitespace_around_verdict(json_value: str, expected: str) -> None: + # A judge returning a valid verdict with incidental surrounding whitespace + # (e.g. a trailing newline) must not be rejected as out-of-domain. + handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler()) + + score = handler.parse( + response_text=f'{{"score_value": {json_value}, "rationale": "test"}}', + scorer_identifier=SCORER_IDENTIFIER, + scored_prompt_id="test-id", + ) + + assert score.raw_score_value == expected + + def test_true_false_response_handler_rejects_value_outside_domain() -> None: handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())