Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 55 additions & 51 deletions markdown_it/rules_core/smartquotes.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,23 @@ def replaceAt(string: str, index: int, ch: str) -> str:
return string[:index] + ch + string[index + 1 :]


def applyReplacements(string: str, replacements: list[tuple[int, str]]) -> str:
"""Replace the characters at the given positions in one pass."""
result: list[str] = []
lastPos = 0
for pos, ch in sorted(replacements):
result.append(string[lastPos:pos])
result.append(ch)
lastPos = pos + 1
result.append(string[lastPos:])
return "".join(result)


def process_inlines(tokens: list[Token], state: StateCore) -> None:
stack: list[dict[str, Any]] = []
# token index -> replacements in that token's original content, applied at
# the end: rebuilding the content for every quote is quadratic
replacements: dict[int, list[tuple[int, str]]] = {}

for i, token in enumerate(tokens):
thisLevel = token.level
Expand All @@ -36,7 +51,7 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None:
# Subtract 1 to get the same index as the js version.
j -= 1

stack = stack[: j + 1]
del stack[j + 1 :]

if token.type != "text":
continue
Expand All @@ -46,22 +61,20 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None:
maximum = len(text)

while pos < maximum:
goto_outer = False
lastIndex = pos
t = QUOTE_RE.search(text[lastIndex:])
t = QUOTE_RE.search(text, pos)
if not t:
break

canOpen = canClose = True
pos = t.start(0) + lastIndex + 1
pos = t.start(0) + 1
isSingle = t.group(0) == "'"

# Find previous character,
# default to space if it's the beginning of the line
lastChar: int | None = 0x20

if t.start(0) + lastIndex - 1 >= 0:
lastChar = charCodeAt(text, t.start(0) + lastIndex - 1)
if t.start(0) - 1 >= 0:
lastChar = charCodeAt(text, t.start(0) - 1)
else:
for j in range(i)[::-1]:
if tokens[j].type == "softbreak" or tokens[j].type == "hardbreak":
Expand Down Expand Up @@ -131,64 +144,55 @@ def process_inlines(tokens: list[Token], state: StateCore) -> None:
if not canOpen and not canClose:
# middle of word
if isSingle:
token.content = replaceAt(
token.content, t.start(0) + lastIndex, APOSTROPHE
)
replacements.setdefault(i, []).append((t.start(0), APOSTROPHE))
continue

if canClose:
# this could be a closing quote, rewind the stack to get a match
for j in range(len(stack))[::-1]:
if canClose and stack:
# this could be a closing quote, find the innermost opener of the
# same kind. Levels on the stack never decrease towards the top and
# never exceed thisLevel, so that is where a rewind of the stack
# would stop; "other" jumps to it instead of rescanning the stack
# for every unmatched closer.
j = len(stack) - 1
if stack[j]["single"] != isSingle:
j = stack[j]["other"]
if j >= 0 and stack[j]["level"] == thisLevel:
item = stack[j]
if stack[j]["level"] < thisLevel:
break
if item["single"] == isSingle and stack[j]["level"] == thisLevel:
item = stack[j]

if isSingle:
openQuote = state.md.options.quotes[2]
closeQuote = state.md.options.quotes[3]
else:
openQuote = state.md.options.quotes[0]
closeQuote = state.md.options.quotes[1]

# replace token.content *before* tokens[item.token].content,
# because, if they are pointing at the same token, replaceAt
# could mess up indices when quote length != 1
token.content = replaceAt(
token.content, t.start(0) + lastIndex, closeQuote
)
tokens[item["token"]].content = replaceAt(
tokens[item["token"]].content, item["pos"], openQuote
)

pos += len(closeQuote) - 1
if item["token"] == i:
pos += len(openQuote) - 1

text = token.content
maximum = len(text)

stack = stack[:j]
goto_outer = True
break
if goto_outer:
goto_outer = False

if isSingle:
openQuote = state.md.options.quotes[2]
closeQuote = state.md.options.quotes[3]
else:
openQuote = state.md.options.quotes[0]
closeQuote = state.md.options.quotes[1]

replacements.setdefault(i, []).append((t.start(0), closeQuote))
replacements.setdefault(item["token"], []).append(
(item["pos"], openQuote)
)

del stack[j:]
continue

if canOpen:
# index of the nearest opener below this one of the other kind
other = len(stack) - 1
if stack and stack[-1]["single"] == isSingle:
other = stack[-1]["other"]
stack.append(
{
"token": i,
"pos": t.start(0) + lastIndex,
"pos": t.start(0),
"single": isSingle,
"level": thisLevel,
"other": other,
}
)
elif canClose and isSingle:
token.content = replaceAt(
token.content, t.start(0) + lastIndex, APOSTROPHE
)
replacements.setdefault(i, []).append((t.start(0), APOSTROPHE))

for i, items in replacements.items():
tokens[i].content = applyReplacements(tokens[i].content, items)


def smartquotes(state: StateCore) -> None:
Expand Down
34 changes: 34 additions & 0 deletions tests/test_api/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,40 @@ def test_inline_rules_do_not_slice_remaining_source(preset, options, src):
assert max(counting.slice_lengths, default=0) <= 1


def test_smartquotes_does_not_copy_text_per_quote():
"""`smartquotes` must not copy the text token for every quote.

It used to search ``text[pos:]`` for the next quote and rebuild the
content after each replacement, both O(len) per quote and so O(n^2) over
a block. Replacements are now applied once per token, slicing each
character at most once.
"""
from markdown_it.rules_core.smartquotes import process_inlines
from markdown_it.rules_core.state_core import StateCore

md = MarkdownIt("commonmark", {"typographer": True})
src = 'a\'b "c" ' * 1_000
counting = _SliceCountingStr(src)
token = Token("text", "", 0, content=counting)
process_inlines([token], StateCore("", md, {}))
assert sum(counting.slice_lengths) <= len(src)
assert token.content == "a\u2019b \u201cc\u201d " * 1_000


def test_smartquotes_unmatched_closers_render_correctly():
"""A closer with no opener of its kind leaves the other kind's openers
open, however many there are.

Each such closer used to rescan the whole opener stack, which made this
input quadratic (seconds for a ~60 kB block).
"""
md = MarkdownIt("commonmark", {"typographer": True}).enable("smartquotes")
n = 10_000
src = '"a ' * n + "b' " * n + 'c"'
expected = "&quot;a " * (n - 1) + "\u201ca " + "b\u2019 " * n + "c\u201d"
assert md.renderInline(src) == expected


def test_pending_appends_do_not_materialise():
"""`append_pending` must be amortised O(1): it buffers fragments and only
builds the string when `pending` is read.
Expand Down