From b2a13cc4b012c767af78702f28c2bef46ea9c4d5 Mon Sep 17 00:00:00 2001 From: Affan Ziya Date: Thu, 19 Nov 2026 23:02:53 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Fix=20quadratic-time?= =?UTF-8?q?=20smartquotes=20on=20blocks=20with=20many=20quotes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port markdown-it's deferred replacements (9ce2087) and remove the other O(n) work done per quote or per token in process_inlines: the text[pos:] search, the stack copy per token, and the full-stack rescan for a closer with no opener of its kind. --- markdown_it/rules_core/smartquotes.py | 106 +++++++++++++------------- tests/test_api/test_main.py | 34 +++++++++ 2 files changed, 89 insertions(+), 51 deletions(-) diff --git a/markdown_it/rules_core/smartquotes.py b/markdown_it/rules_core/smartquotes.py index 3392dcb5..e5d988ed 100644 --- a/markdown_it/rules_core/smartquotes.py +++ b/markdown_it/rules_core/smartquotes.py @@ -21,8 +21,23 @@ def replaceAt(string: str, index: int, ch: str) -> str: return string[:index] + ch + string[index + 1 :] +def applyReplacements(string: str, replacements: list[tuple[int, str]]) -> str: + """Replace the characters at the given positions in one pass.""" + result: list[str] = [] + lastPos = 0 + for pos, ch in sorted(replacements): + result.append(string[lastPos:pos]) + result.append(ch) + lastPos = pos + 1 + result.append(string[lastPos:]) + return "".join(result) + + def process_inlines(tokens: list[Token], state: StateCore) -> None: stack: list[dict[str, Any]] = [] + # token index -> replacements in that token's original content, applied at + # the end: rebuilding the content for every quote is quadratic + replacements: dict[int, list[tuple[int, str]]] = {} for i, token in enumerate(tokens): thisLevel = token.level @@ -36,7 +51,7 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None: # Subtract 1 to get the same index as the js version. j -= 1 - stack = stack[: j + 1] + del stack[j + 1 :] if token.type != "text": continue @@ -46,22 +61,20 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None: maximum = len(text) while pos < maximum: - goto_outer = False - lastIndex = pos - t = QUOTE_RE.search(text[lastIndex:]) + t = QUOTE_RE.search(text, pos) if not t: break canOpen = canClose = True - pos = t.start(0) + lastIndex + 1 + pos = t.start(0) + 1 isSingle = t.group(0) == "'" # Find previous character, # default to space if it's the beginning of the line lastChar: int | None = 0x20 - if t.start(0) + lastIndex - 1 >= 0: - lastChar = charCodeAt(text, t.start(0) + lastIndex - 1) + if t.start(0) - 1 >= 0: + lastChar = charCodeAt(text, t.start(0) - 1) else: for j in range(i)[::-1]: if tokens[j].type == "softbreak" or tokens[j].type == "hardbreak": @@ -131,64 +144,55 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None: if not canOpen and not canClose: # middle of word if isSingle: - token.content = replaceAt( - token.content, t.start(0) + lastIndex, APOSTROPHE - ) + replacements.setdefault(i, []).append((t.start(0), APOSTROPHE)) continue - if canClose: - # this could be a closing quote, rewind the stack to get a match - for j in range(len(stack))[::-1]: + if canClose and stack: + # this could be a closing quote, find the innermost opener of the + # same kind. Levels on the stack never decrease towards the top and + # never exceed thisLevel, so that is where a rewind of the stack + # would stop; "other" jumps to it instead of rescanning the stack + # for every unmatched closer. + j = len(stack) - 1 + if stack[j]["single"] != isSingle: + j = stack[j]["other"] + if j >= 0 and stack[j]["level"] == thisLevel: item = stack[j] - if stack[j]["level"] < thisLevel: - break - if item["single"] == isSingle and stack[j]["level"] == thisLevel: - item = stack[j] - - if isSingle: - openQuote = state.md.options.quotes[2] - closeQuote = state.md.options.quotes[3] - else: - openQuote = state.md.options.quotes[0] - closeQuote = state.md.options.quotes[1] - - # replace token.content *before* tokens[item.token].content, - # because, if they are pointing at the same token, replaceAt - # could mess up indices when quote length != 1 - token.content = replaceAt( - token.content, t.start(0) + lastIndex, closeQuote - ) - tokens[item["token"]].content = replaceAt( - tokens[item["token"]].content, item["pos"], openQuote - ) - - pos += len(closeQuote) - 1 - if item["token"] == i: - pos += len(openQuote) - 1 - - text = token.content - maximum = len(text) - - stack = stack[:j] - goto_outer = True - break - if goto_outer: - goto_outer = False + + if isSingle: + openQuote = state.md.options.quotes[2] + closeQuote = state.md.options.quotes[3] + else: + openQuote = state.md.options.quotes[0] + closeQuote = state.md.options.quotes[1] + + replacements.setdefault(i, []).append((t.start(0), closeQuote)) + replacements.setdefault(item["token"], []).append( + (item["pos"], openQuote) + ) + + del stack[j:] continue if canOpen: + # index of the nearest opener below this one of the other kind + other = len(stack) - 1 + if stack and stack[-1]["single"] == isSingle: + other = stack[-1]["other"] stack.append( { "token": i, - "pos": t.start(0) + lastIndex, + "pos": t.start(0), "single": isSingle, "level": thisLevel, + "other": other, } ) elif canClose and isSingle: - token.content = replaceAt( - token.content, t.start(0) + lastIndex, APOSTROPHE - ) + replacements.setdefault(i, []).append((t.start(0), APOSTROPHE)) + + for i, items in replacements.items(): + tokens[i].content = applyReplacements(tokens[i].content, items) def smartquotes(state: StateCore) -> None: diff --git a/tests/test_api/test_main.py b/tests/test_api/test_main.py index 74c7b0f8..968df659 100644 --- a/tests/test_api/test_main.py +++ b/tests/test_api/test_main.py @@ -498,6 +498,40 @@ def test_inline_rules_do_not_slice_remaining_source(preset, options, src): assert max(counting.slice_lengths, default=0) <= 1 +def test_smartquotes_does_not_copy_text_per_quote(): + """`smartquotes` must not copy the text token for every quote. + + It used to search ``text[pos:]`` for the next quote and rebuild the + content after each replacement, both O(len) per quote and so O(n^2) over + a block. Replacements are now applied once per token, slicing each + character at most once. + """ + from markdown_it.rules_core.smartquotes import process_inlines + from markdown_it.rules_core.state_core import StateCore + + md = MarkdownIt("commonmark", {"typographer": True}) + src = 'a\'b "c" ' * 1_000 + counting = _SliceCountingStr(src) + token = Token("text", "", 0, content=counting) + process_inlines([token], StateCore("", md, {})) + assert sum(counting.slice_lengths) <= len(src) + assert token.content == "a\u2019b \u201cc\u201d " * 1_000 + + +def test_smartquotes_unmatched_closers_render_correctly(): + """A closer with no opener of its kind leaves the other kind's openers + open, however many there are. + + Each such closer used to rescan the whole opener stack, which made this + input quadratic (seconds for a ~60 kB block). + """ + md = MarkdownIt("commonmark", {"typographer": True}).enable("smartquotes") + n = 10_000 + src = '"a ' * n + "b' " * n + 'c"' + expected = ""a " * (n - 1) + "\u201ca " + "b\u2019 " * n + "c\u201d" + assert md.renderInline(src) == expected + + def test_pending_appends_do_not_materialise(): """`append_pending` must be amortised O(1): it buffers fragments and only builds the string when `pending` is read.