From 0276f579d05acdbb7eef52563e3f954256469bfa Mon Sep 17 00:00:00 2001
From: WatchTree-19 <119982314+WatchTree-19@users.noreply.github.com>
Date: Thu, 13 Aug 2026 13:43:26 +0100
Subject: [PATCH] fix(scorer): PlagiarismScorer verbatim fast path matched
 sub-word substrings

The verbatim-match shortcut in _plagiarism_score used a raw string check
(reference in response). The rest of the scorer is word-level: it tokenizes
with lowercasing and punctuation removal before computing LCS / Levenshtein /
Jaccard. The raw check was inconsistent with that in both directions:

  - false positive: a short reference that is only a substring of a longer
    response word scored 1.0 (e.g. reference 'cat' vs response 'concatenate
    the results' returned full plagiarism for every metric).
  - missed match: a word-level verbatim copy differing only in case or
    punctuation did not take the fast path.

Compare the tokenized sequences instead, so the fast path matches the same
word-level semantics the metrics use. Adds regression tests.
---
 pyrit/score/float_scale/plagiarism_scorer.py | 23 ++++++++++++++--
 tests/unit/score/test_plagiarism_scorer.py   | 28 ++++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py
index b579418bdc..3e78a2daf3 100644
--- a/pyrit/score/float_scale/plagiarism_scorer.py
+++ b/pyrit/score/float_scale/plagiarism_scorer.py
@@ -124,6 +124,20 @@ def _ngram_set(self, tokens: list[str], n: int) -> set[tuple[str, ...]]:
         """
         return {tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)}
 
+    def _is_contiguous_sublist(self, sub: list[str], full: list[str]) -> bool:
+        """
+        Check whether ``sub`` appears as a contiguous run of tokens inside ``full``.
+
+        This mirrors the word-level tokenization the metrics rely on, so the
+        verbatim-match fast path stays consistent with them.
+
+        Returns:
+            bool: True if ``sub`` is a contiguous sublist of ``full``.
+        """
+        if not sub or len(sub) > len(full):
+            return False
+        return any(full[i : i + len(sub)] == sub for i in range(len(full) - len(sub) + 1))
+
     def _plagiarism_score(
         self,
         response: str,
@@ -139,8 +153,13 @@ def _plagiarism_score(
         if response_len == 0 or reference_len == 0:
             return 0.0
 
-        # If reference is in response, all three metrics should be 1.0
-        if reference in response:
+        # If the reference appears verbatim (word-level) in the response, all
+        # three metrics should be 1.0. Compare tokenized sequences rather than
+        # raw strings so this fast path matches the case/punctuation-insensitive
+        # tokenization used below, and so a short reference that is merely a
+        # substring of a longer response word (e.g. "cat" in "concatenate")
+        # does not falsely score as fully plagiarized.
+        if self._is_contiguous_sublist(tokens_reference, tokens_response):
             return 1.0
 
         # Compute the LCS metric (normalized by reference length)
diff --git a/tests/unit/score/test_plagiarism_scorer.py b/tests/unit/score/test_plagiarism_scorer.py
index aef799eea6..78826a3c37 100644
--- a/tests/unit/score/test_plagiarism_scorer.py
+++ b/tests/unit/score/test_plagiarism_scorer.py
@@ -375,6 +375,34 @@ def test_plagiarism_score_jaccard_reference_contained_in_response(self, scorer):
         score = scorer._plagiarism_score(response, reference, metric=PlagiarismMetric.JACCARD, n=3)
         assert score == 1.0  # Should be perfect match when reference is contained
 
+    def test_plagiarism_score_reference_substring_of_word_not_plagiarism(self, scorer):
+        """A reference that is only a substring of a longer response word is not plagiarism.
+
+        The verbatim-match fast path must operate on word-level tokens, not raw
+        characters. Otherwise a short reference such as "cat" would falsely score
+        1.0 against a response containing "concatenate".
+        """
+        reference = "cat"
+        response = "concatenate the results"
+        for metric in PlagiarismMetric:
+            score = scorer._plagiarism_score(response, reference, metric=metric)
+            assert score == 0.0, f"{metric.value} should not treat a sub-word match as plagiarism"
+
+    def test_plagiarism_score_verbatim_match_ignores_case_and_punctuation(self, scorer):
+        """The verbatim fast path should still fire across case and punctuation differences."""
+        reference = "The Secret Plan"
+        response = "the secret plan!"
+        for metric in PlagiarismMetric:
+            score = scorer._plagiarism_score(response, reference, metric=metric)
+            assert score == 1.0, f"{metric.value} should treat a word-level verbatim copy as plagiarism"
+
+    def test_is_contiguous_sublist(self, scorer):
+        """Directly exercise the tokenized sublist helper."""
+        assert scorer._is_contiguous_sublist(["b", "c"], ["a", "b", "c", "d"]) is True
+        assert scorer._is_contiguous_sublist(["a", "c"], ["a", "b", "c"]) is False
+        assert scorer._is_contiguous_sublist([], ["a"]) is False
+        assert scorer._is_contiguous_sublist(["a", "b"], ["a"]) is False
+
 
 class TestPlagiarismMetricEnum:
     """Test cases for the PlagiarismMetric enum."""