diff --git a/changelog/14523.improvement.rst b/changelog/14523.improvement.rst new file mode 100644 index 00000000000..d53374c62df --- /dev/null +++ b/changelog/14523.improvement.rst @@ -0,0 +1,13 @@ +Assertion explanations are now built lazily and the truncator stops +the comparison helpers as soon as it has enough output, so comparing +two large collections no longer builds the full diff in order to +discard it. A focused micro-benchmark the worst case scenario +(``set(range(500_000)) == set(range(1, 500_001))``) drops from ~2,200 ms +to ~43 ms; but realistic test suite with mostly small diffs should be +unchanged. + +The truncation footer no longer reports the hidden-line count +(``...Full output truncated (N lines hidden), ...`` becomes +``...Full output truncated, ...``); diff lines now carry a redundant +``\x1b[0m`` reset prefix (invisible to terminals) so we can handle +line one by one. diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 29ba190b7e7..2e1681cd9cd 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -148,7 +148,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E 1 E 1... E - E ...Full output truncated (7 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show failure_demo.py:62: AssertionError _________________ TestSpecialisedExplanations.test_eq_list _________________ diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index db36a5a7206..752d0206526 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -172,7 +172,7 @@ Now we can increase pytest's verbosity: E 'banana', E 'apple',... E - E ...Full output truncated (7 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show test_verbosity_example.py:8: AssertionError ____________________________ test_numbers_fail _____________________________ @@ -190,7 +190,7 @@ Now we can increase pytest's verbosity: E {'10': 10, '20': 20, '30': 30, '40': 40} E ... E - E ...Full output truncated (16 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show test_verbosity_example.py:14: AssertionError ___________________________ test_long_text_fail ____________________________ diff --git a/src/_pytest/_io/pprint.py b/src/_pytest/_io/pprint.py index ec41b449ddf..ae58c34c753 100644 --- a/src/_pytest/_io/pprint.py +++ b/src/_pytest/_io/pprint.py @@ -56,6 +56,51 @@ def _safe_tuple(t): return _safe_key(t[0]), _safe_key(t[1]) +class _LineBudgetExceeded(Exception): + """Internal: signals that ``_LineBudgetStream`` has reached its cap. + + Raised from inside ``stream.write`` so the formatter's recursion + unwinds at the next write; the caller catches it and uses whatever + output accumulated so far. + """ + + +class _LineBudgetStream: + r"""Stream that collects ``write`` calls into lines and bails out + once ``max_lines`` have been produced. + + Lets ``PrettyPrinter._format`` stop early on huge collections: the + formatter writes one ``\n``-terminated chunk per element, so the + budget check fires on element boundaries. + """ + + __slots__ = ("_lines", "_max", "_pending") + + def __init__(self, max_lines: int) -> None: + self._lines: list[str] = [] + self._pending: list[str] = [] + self._max = max_lines + + def write(self, s: str) -> None: + if "\n" not in s: + if s: + self._pending.append(s) + return + parts = s.split("\n") + self._pending.append(parts[0]) + self._lines.append("".join(self._pending)) + self._lines.extend(parts[1:-1]) + self._pending = [parts[-1]] if parts[-1] else [] + if len(self._lines) >= self._max: + raise _LineBudgetExceeded + + def finish(self) -> list[str]: + if self._pending: + self._lines.append("".join(self._pending)) + self._pending = [] + return self._lines + + class PrettyPrinter: def __init__( self, @@ -91,6 +136,48 @@ def pformat(self, object: Any) -> str: self._format(object, sio, 0, 0, set(), 0) return sio.getvalue() + def pformat_lines(self, object: Any, max_lines: int | None = None) -> list[str]: + """Pretty-print ``object`` and return its lines. + + With ``max_lines=None`` this is just ``self.pformat(object). + splitlines()`` — the fast C path through ``StringIO`` and + ``str.splitlines``. + + With ``max_lines`` set, the formatter is wired to a Python-level + stream that raises once that many lines have been produced; the + caller gets back whatever was emitted before the abort. For + huge collections this turns an O(N) pformat into O(``max_lines``) + — useful when a downstream truncator is going to drop everything + past a small budget anyway. + + Sized containers small enough that pformat will obviously fit + in the budget take the fast path too: the pure-Python budget + stream is ~1.3x slower than ``StringIO`` + ``splitlines`` on + tiny inputs (per-write overhead dominates), and paying that on + common-case small assertion diffs is a net loss. + """ + if max_lines is None: + return self.pformat(object).splitlines() + # Sufficient condition for "the budget will never fire": one + # pformat line per element is the lower bound for any + # container, so ``len(object) <= max_lines`` guarantees the + # full pformat fits in the budget. + try: + size = len(object) + except TypeError: + size = -1 + if 0 <= size <= max_lines: + return self.pformat(object).splitlines() + stream = _LineBudgetStream(max_lines) + try: + # ``_format``'s ``IO[str]`` annotation is overly tight — it + # only ever calls ``stream.write(str)``, which is all this + # budget-aware stream implements. + self._format(object, stream, 0, 0, set(), 0) # type: ignore[arg-type] + except _LineBudgetExceeded: + pass + return stream.finish() + def _format( self, object: Any, @@ -236,7 +323,12 @@ def _pprint_set( else: stream.write(typ.__name__ + "({") endchar = "})" - object = sorted(object, key=_safe_key) + try: + object = sorted(object) + except TypeError: + # Heterogeneous element types — fall back to a key that + # tolerates unorderable pairs by string-comparing their types. + object = sorted(object, key=_safe_key) self._format_items(object, stream, indent, allowance, context, level) stream.write(endchar) diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index a4530192407..847dbaa287a 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -181,13 +181,21 @@ def callbinrepr(op, left: object, right: object) -> str | None: config=item.config, op=op, left=left, right=right ) for new_expl in hook_result: + # Plugin-supplied lists are truncated here; the built-in impl + # already truncates as it streams, so re-applying truncation + # to its output is a near no-op (the body fits the budget, + # only the footer line is re-emitted with the same wording). + # ``materialize_with_truncation`` can return ``[]`` when the + # input was a truthy-but-empty iterable, so re-check after + # materialising. if new_expl: - new_expl = truncate.truncate_if_required(new_expl, item) - new_expl = [line.replace("\n", "\\n") for line in new_expl] - res = "\n~".join(new_expl) - if item.config.getvalue("assertmode") == "rewrite": - res = res.replace("%", "%%") - return res + new_expl = truncate.materialize_with_truncation(new_expl, item.config) + if new_expl: + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + if item.config.getvalue("assertmode") == "rewrite": + res = res.replace("%", "%%") + return res return None saved_assert_hooks = util._reprcompare, util._assertion_pass @@ -218,16 +226,40 @@ def pytest_sessionfinish(session: Session) -> None: def pytest_assertrepr_compare( config: Config, op: str, left: Any, right: Any ) -> list[str] | None: + """Return an explanation for ``left op right``. + + Internally ``util.assertrepr_compare`` is a generator; we feed it + through ``materialize_with_truncation`` so a huge comparison + short-circuits at the truncation threshold without building the + full diff, while still returning the ``list[str] | None`` shape + the hook spec advertises. + """ if config.pluginmanager.has_plugin("terminalreporter"): highlighter = config.get_terminal_writer()._highlight else: # Keep it plaintext when not using terminalrepoterer (#14377). highlighter = util.dummy_highlighter - return util.assertrepr_compare( + # When truncation is going to clip the explanation downstream, tell + # the comparison helpers to cap their pformat output at the same + # budget so they don't spend O(N) formatting lines we're about to + # drop. ``+ 3`` matches the truncator's own ``line_cap``: 2 lines + # for the truncation message it appends (blank + footer) plus 1 + # for overshoot detection. ``difflib.ndiff`` over two K-line + # pformat outputs produces at least K output lines (more when the + # sides differ), and the truncator pulls at most ``trunc_lines + + # 3`` lines from the whole explanation, so a per-side pformat + # budget of ``trunc_lines + 3`` covers the worst case. With + # truncation disabled the cap stays ``None`` and the user gets the + # full diff. + should_truncate, trunc_lines, _ = truncate._get_truncation_parameters(config) + pformat_cap = trunc_lines + 3 if should_truncate and trunc_lines > 0 else None + lines = util.assertrepr_compare( op=op, left=left, right=right, verbose=config.get_verbosity(Config.VERBOSITY_ASSERTIONS), highlighter=highlighter, assertion_text_diff_style=util.get_assertion_text_diff_style(config), + pformat_cap=pformat_cap, ) + return truncate.materialize_with_truncation(lines, config) or None diff --git a/src/_pytest/assertion/_compare_any.py b/src/_pytest/assertion/_compare_any.py index 27556c2e8db..73012abc416 100644 --- a/src/_pytest/assertion/_compare_any.py +++ b/src/_pytest/assertion/_compare_any.py @@ -28,26 +28,30 @@ def _compare_eq_any( highlighter: _HighlightFunc, verbose: int, assertion_text_diff_style: _AssertionTextDiffStyle, -) -> list[str]: - explanation = [] + pformat_cap: int | None = None, +) -> Iterator[str]: + """Yield the per-line explanation for ``left == right`` (without summary). + + Yields nothing when no specialised explanation applies, so consumers + can stream the output and bail out early (e.g. for truncation) without + materialising the entire diff first. + """ if istext(left) and istext(right): - explanation = list( - _compare_eq_text( - left, - right, - highlighter, - verbose, - assertion_text_diff_style, - ) + yield from _compare_eq_text( + left, + right, + highlighter, + verbose, + assertion_text_diff_style, ) else: from _pytest.python_api import ApproxBase # Although the common order should be obtained == approx(...), allow both ways. if isinstance(right, ApproxBase): - explanation = right._repr_compare(left) + yield from right._repr_compare(left) elif isinstance(left, ApproxBase): - explanation = left._repr_compare(right) + yield from left._repr_compare(right) elif type(left) is type(right) and ( isdatacls(left) or isattrs(left) or isnamedtuple(left) ): @@ -55,27 +59,24 @@ def _compare_eq_any( # field values, not the type or field names. But this branch # intentionally only handles the same-type case, which was often # used in older code bases before dataclasses/attrs were available. - explanation = list( - _compare_eq_cls( - left, - right, - highlighter, - verbose, - assertion_text_diff_style, - ) + yield from _compare_eq_cls( + left, + right, + highlighter, + verbose, + assertion_text_diff_style, ) elif issequence(left) and issequence(right): - explanation = list(_compare_eq_sequence(left, right, highlighter, verbose)) + yield from _compare_eq_sequence(left, right, highlighter, verbose) elif isset(left) and isset(right): - explanation = _compare_eq_set(left, right, highlighter, verbose) + yield from _compare_eq_set(left, right, highlighter, verbose) elif ismapping(left) and ismapping(right): - explanation = list(_compare_eq_mapping(left, right, highlighter, verbose)) + yield from _compare_eq_mapping(left, right, highlighter, verbose) if isiterable(left) and isiterable(right): - expl = _compare_eq_iterable(left, right, highlighter, verbose) - explanation.extend(expl) - - return explanation + yield from _compare_eq_iterable( + left, right, highlighter, verbose, pformat_cap + ) def _compare_eq_cls( diff --git a/src/_pytest/assertion/_compare_sequence.py b/src/_pytest/assertion/_compare_sequence.py index cd0043bf7ce..52edb136d6e 100644 --- a/src/_pytest/assertion/_compare_sequence.py +++ b/src/_pytest/assertion/_compare_sequence.py @@ -15,6 +15,7 @@ def _compare_eq_iterable( right: Iterable[object], highlighter: _HighlightFunc, verbose: int = 0, + pformat_cap: int | None = None, ) -> Iterator[str]: if verbose <= 0 and not running_on_ci(): yield "Use -v to get more diff" @@ -22,19 +23,28 @@ def _compare_eq_iterable( # dynamic import to speedup pytest import difflib - left_formatting = PrettyPrinter().pformat(left).splitlines() - right_formatting = PrettyPrinter().pformat(right).splitlines() + # ``pformat_cap`` is computed by the dispatcher from the + # truncator's ``truncation_limit_lines``: when truncation is going + # to drop everything past that budget anyway, we don't bother + # formatting more. ``None`` means no cap (``-vv`` or CI: the user + # wants the full diff). + pp = PrettyPrinter() + left_formatting = pp.pformat_lines(left, max_lines=pformat_cap) + right_formatting = pp.pformat_lines(right, max_lines=pformat_cap) yield "" yield "Full diff:" # "right" is the expected base against which we compare "left", # see https://github.com/pytest-dev/pytest/issues/3333 - yield from highlighter( - "\n".join( - line.rstrip() for line in difflib.ndiff(right_formatting, left_formatting) - ), - lexer="diff", - ).splitlines() + # + # Yield each ndiff line through the highlighter individually so the + # streaming truncator can stop pulling from ``difflib.ndiff`` as + # soon as its budget is full. The diff lexer is line-oriented, so + # per-line highlighting is equivalent — it just adds a redundant + # ``\x1b[0m`` reset at the start of each line (invisible to the + # terminal). + for line in difflib.ndiff(right_formatting, left_formatting): + yield highlighter(line.rstrip(), lexer="diff") def _compare_eq_sequence( diff --git a/src/_pytest/assertion/_compare_set.py b/src/_pytest/assertion/_compare_set.py index 0fac608fe5c..2817133790d 100644 --- a/src/_pytest/assertion/_compare_set.py +++ b/src/_pytest/assertion/_compare_set.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from collections.abc import Iterator from collections.abc import Set as AbstractSet from typing import TypeAlias @@ -13,14 +14,12 @@ def _set_one_sided_diff( set1: AbstractSet[object], set2: AbstractSet[object], highlighter: _HighlightFunc, -) -> list[str]: - explanation = [] +) -> Iterator[str]: diff = set1 - set2 if diff: - explanation.append(f"Extra items in the {posn} set:") + yield f"Extra items in the {posn} set:" for item in diff: - explanation.append(highlighter(saferepr(item))) - return explanation + yield highlighter(saferepr(item)) def _compare_eq_set( @@ -28,65 +27,73 @@ def _compare_eq_set( right: AbstractSet[object], highlighter: _HighlightFunc, verbose: int = 0, -) -> list[str]: - explanation = [] - explanation.extend(_set_one_sided_diff("left", left, right, highlighter)) - explanation.extend(_set_one_sided_diff("right", right, left, highlighter)) - return explanation +) -> Iterator[str]: + yield from _set_one_sided_diff("left", left, right, highlighter) + yield from _set_one_sided_diff("right", right, left, highlighter) -def _compare_gt_set( +def _compare_gte_set( left: AbstractSet[object], right: AbstractSet[object], highlighter: _HighlightFunc, verbose: int = 0, -) -> list[str]: - explanation = _compare_gte_set(left, right, highlighter) - if not explanation: - return ["Both sets are equal"] - return explanation +) -> Iterator[str]: + yield from _set_one_sided_diff("right", right, left, highlighter) -def _compare_lt_set( +def _compare_lte_set( left: AbstractSet[object], right: AbstractSet[object], highlighter: _HighlightFunc, verbose: int = 0, -) -> list[str]: - explanation = _compare_lte_set(left, right, highlighter) - if not explanation: - return ["Both sets are equal"] - return explanation +) -> Iterator[str]: + yield from _set_one_sided_diff("left", left, right, highlighter) -def _compare_gte_set( +def _compare_gt_set( left: AbstractSet[object], right: AbstractSet[object], highlighter: _HighlightFunc, verbose: int = 0, -) -> list[str]: - return _set_one_sided_diff("right", right, left, highlighter) +) -> Iterator[str]: + if left == right: + yield "Both sets are equal" + else: + yield from _set_one_sided_diff("right", right, left, highlighter) -def _compare_lte_set( +def _compare_lt_set( left: AbstractSet[object], right: AbstractSet[object], highlighter: _HighlightFunc, verbose: int = 0, -) -> list[str]: - return _set_one_sided_diff("left", left, right, highlighter) +) -> Iterator[str]: + if left == right: + yield "Both sets are equal" + else: + yield from _set_one_sided_diff("left", left, right, highlighter) SetComparisonFunction: TypeAlias = Callable[ [AbstractSet[object], AbstractSet[object], _HighlightFunc, int], - list[str], + Iterator[str], ] + +def _both_sets_are_equal( + left: AbstractSet[object], + right: AbstractSet[object], + highlighter: _HighlightFunc, + verbose: int = 0, +) -> Iterator[str]: + yield "Both sets are equal" + + SET_COMPARISON_FUNCTIONS: dict[str, SetComparisonFunction] = { # == can't be done here without a prior refactor because there's an additional # explanation for iterable in _compare_eq_any # "==": _compare_eq_set, - "!=": lambda *a, **kw: ["Both sets are equal"], + "!=": _both_sets_are_equal, ">=": _compare_gte_set, "<=": _compare_lte_set, ">": _compare_gt_set, diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index d62ca33cc4b..e137d8e0c1f 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -6,9 +6,10 @@ from __future__ import annotations +from collections.abc import Iterable + from _pytest.compat import running_on_ci from _pytest.config import Config -from _pytest.nodes import Item DEFAULT_MAX_LINES = 8 @@ -16,32 +17,52 @@ USAGE_MSG = "use '-vv' to show" -def truncate_if_required(explanation: list[str], item: Item) -> list[str]: - """Truncate this assertion explanation if the given test item is eligible.""" - should_truncate, max_lines, max_chars = _get_truncation_parameters(item) - if should_truncate: - return _truncate_explanation( - explanation, - max_lines=max_lines, - max_chars=max_chars, - ) - return explanation +def materialize_with_truncation(lines: Iterable[str], config: Config) -> list[str]: + """Materialise a streaming explanation, applying truncation lazily. + + Pulls from ``lines`` only until the truncation threshold is reached; + once exceeded, the rest of the iterator is dropped without being + consumed. This lets a huge comparison short-circuit instead of + building (and immediately discarding) megabytes of explanation text. + """ + should_truncate, max_lines, max_chars = _get_truncation_parameters(config) + if not should_truncate: + return list(lines) + + tolerable_max_chars = max_chars + 70 + # Pull just past max_lines so ``_truncate_explanation`` can detect the + # overflow without us materialising more than we need. + line_cap = max_lines + 3 if max_lines > 0 else None + buffered: list[str] = [] + char_count = 0 + for line in lines: + buffered.append(line) + char_count += len(line) + if line_cap is not None and len(buffered) >= line_cap: + break + if max_chars > 0 and char_count > tolerable_max_chars: + break + else: + # Iterator exhausted within limits — nothing to truncate. + return buffered + + return _truncate_explanation(buffered, max_lines=max_lines, max_chars=max_chars) -def _get_truncation_parameters(item: Item) -> tuple[bool, int, int]: - """Return the truncation parameters related to the given item, as (should truncate, max lines, max chars).""" +def _get_truncation_parameters(config: Config) -> tuple[bool, int, int]: + """Return the truncation parameters from the given config, as (should truncate, max lines, max chars).""" # We do not need to truncate if one of conditions is met: # 1. Verbosity level is 2 or more; # 2. Test is being run in CI environment; # 3. Both truncation_limit_lines and truncation_limit_chars # .ini parameters are set to 0 explicitly. - max_lines = item.config.getini("truncation_limit_lines") + max_lines = config.getini("truncation_limit_lines") max_lines = int(max_lines if max_lines is not None else DEFAULT_MAX_LINES) - max_chars = item.config.getini("truncation_limit_chars") + max_chars = config.getini("truncation_limit_chars") max_chars = int(max_chars if max_chars is not None else DEFAULT_MAX_CHARS) - verbose = item.config.get_verbosity(Config.VERBOSITY_ASSERTIONS) + verbose = config.get_verbosity(Config.VERBOSITY_ASSERTIONS) should_truncate = verbose < 2 and not running_on_ci() should_truncate = should_truncate and (max_lines > 0 or max_chars > 0) @@ -66,20 +87,9 @@ def _truncate_explanation( When this function is launched we know max_lines > 0 or max_chars > 0 because _get_truncation_parameters was called first. """ - # The length of the truncation explanation depends on the number of lines - # removed but is at least 68 characters: - # The real value is - # 64 (for the base message: - # '...\n...Full output truncated (1 line hidden), use '-vv' to show")' - # ) - # + 1 (for plural) - # + int(math.log10(len(input_lines) - max_lines)) (number of hidden line, at least 1) - # + 3 for the '...' added to the truncated line - # But if there's more than 100 lines it's very likely that we're going to - # truncate, so we don't need the exact value using log10. - tolerable_max_chars = ( - max_chars + 70 # 64 + 1 (for plural) + 2 (for '99') + 3 for '...' - ) + # Slack on top of ``max_chars`` so a body that just fits the budget + # doesn't get truncated solely to make room for the footer. + tolerable_max_chars = max_chars + 70 # The truncation explanation add two lines to the output if max_lines == 0 or len(input_lines) <= max_lines + 2: if max_chars == 0 or sum(len(s) for s in input_lines) <= tolerable_max_chars: @@ -89,24 +99,19 @@ def _truncate_explanation( # Truncate first to max_lines, and then truncate to max_chars if necessary truncated_explanation = input_lines[:max_lines] # We reevaluate the need to truncate chars following removal of some lines - need_to_truncate_char = ( + if ( max_chars > 0 and sum(len(e) for e in truncated_explanation) > tolerable_max_chars - ) - if need_to_truncate_char: + ): truncated_explanation = _truncate_by_char_count( truncated_explanation, max_chars ) # Something was truncated, adding '...' at the end to show that truncated_explanation[-1] += "..." - truncated_line_count = ( - len(input_lines) - len(truncated_explanation) + int(need_to_truncate_char) - ) return [ *truncated_explanation, "", - f"...Full output truncated ({truncated_line_count} line" - f"{'' if truncated_line_count == 1 else 's'} hidden), {USAGE_MSG}", + f"...Full output truncated, {USAGE_MSG}", ] diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index f6fe2a7e8f8..e53b290744e 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections.abc import Callable +from collections.abc import Iterator from collections.abc import Sequence from typing import Literal from unicodedata import normalize @@ -139,8 +140,20 @@ def assertrepr_compare( verbose: int, highlighter: _HighlightFunc, assertion_text_diff_style: _AssertionTextDiffStyle, -) -> list[str] | None: - """Return specialised explanations for some operators/operands.""" + pformat_cap: int | None = None, +) -> Iterator[str]: + """Yield specialised explanations for some operators/operands. + + The first line yielded is always the summary (``left op right``); + subsequent lines are the per-line explanation. Yields nothing when no + specialised explanation applies, which lets consumers map an empty + iterator to "no explanation" without materialising anything. + + The iterator is lazy on purpose: a streaming consumer (e.g. the + truncator in ``pytest_assertrepr_compare``) can stop pulling lines as + soon as it has enough to show, so an enormous diff doesn't have to be + built in full just to be thrown away. + """ # Strings which normalize equal are often hard to distinguish when printed; use ascii() to make this easier. # See issue #3246. use_ascii = ( @@ -164,37 +177,40 @@ def assertrepr_compare( summary = f"{left_repr} {op} {right_repr}" - explanation = None + summary_yielded = False try: if op == "==": - explanation = _compare_eq_any( + source = _compare_eq_any( left, right, highlighter, verbose, assertion_text_diff_style, + pformat_cap, ) - elif op == "not in": - if istext(left) and istext(right): - explanation = list(_notin_text(left, right, verbose)) - elif op in {"!=", ">=", "<=", ">", "<"}: - if isset(left) and isset(right): - explanation = SET_COMPARISON_FUNCTIONS[op]( - left, right, highlighter, verbose - ) - + elif op == "not in" and istext(left) and istext(right): + source = _notin_text(left, right, verbose) + elif op in {"!=", ">=", "<=", ">", "<"} and isset(left) and isset(right): + source = SET_COMPARISON_FUNCTIONS[op](left, right, highlighter, verbose) + else: + source = iter(()) + + for line in source: + if not summary_yielded: + yield summary + if line != "": + yield "" + summary_yielded = True + yield line except outcomes.Exit: raise except Exception: repr_crash = _pytest._code.ExceptionInfo.from_current()._getreprcrash() - explanation = [ - f"(pytest_assertion plugin: representation of details failed: {repr_crash}.", - " Probably an object has a faulty __repr__.)", - ] - - if not explanation: - return None - - if explanation[0] != "": - explanation = ["", *explanation] - return [summary, *explanation] + if not summary_yielded: + yield summary + yield "" + summary_yielded = True + yield ( + f"(pytest_assertion plugin: representation of details failed: {repr_crash}." + ) + yield " Probably an object has a faulty __repr__.)" diff --git a/testing/python/approx.py b/testing/python/approx.py index 88d46cbb755..c5ca03fe823 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -313,7 +313,7 @@ def test_error_messages_with_different_verbosity(self, assert_approx_raises_rege rf"^ \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}$", rf"^ \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}\.\.\.$", "^ $", - rf"^ ...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show$", + r"^ ...Full output truncated, use '-vv' to show$", ], verbosity_level=0, ) diff --git a/testing/test_assertion.py b/testing/test_assertion.py index c25487bdf33..95e31332c0c 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -56,6 +56,11 @@ def get_verbosity(self, verbosity_type: str | None = None) -> int: def getini(self, name: str) -> str: if name == util.ASSERTION_TEXT_DIFF_STYLE_INI: return assertion_text_diff_style + # Disable truncation so ``callop``-style tests can compare + # against the full explanation. Dedicated truncation tests + # use their own config in :class:`TestTruncateMaterialize`. + if name in ("truncation_limit_lines", "truncation_limit_chars"): + return "0" raise KeyError(f"Not mocked out: {name}") return Config() @@ -1043,7 +1048,9 @@ def __repr__(self): assert expl is not None assert expl[0].startswith("{} == <[ValueError") assert "raised in repr" in expl[0] - assert expl[2:] == [ + # Streaming explanation: any per-line output produced before the + # bad repr is preserved, then the failure notice is appended. + assert expl[-2:] == [ "(pytest_assertion plugin: representation of details failed:" f" {__file__}:{A.__repr__.__code__.co_firstlineno + 1}: ValueError: 42.", " Probably an object has a faulty __repr__.)", @@ -1152,7 +1159,7 @@ def test_recursive_dataclasses(self, pytester: Pytester) -> None: "E Drill down into differing attribute g:", "E g: S(a=10, b='ten') != S(a=20, b='xxx')...", "E ", - "E ...Full output truncated (51 lines hidden), use '-vv' to show", + "E ...Full output truncated, use '-vv' to show", ], consecutive=True, ) @@ -1525,7 +1532,6 @@ def test_truncates_at_8_lines_when_given_list_of_empty_strings(self) -> None: assert result != expl assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "42 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1536,7 +1542,6 @@ def test_truncates_at_8_lines_when_first_8_lines_are_LT_max_chars(self) -> None: assert result != expl assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert f"{total_lines - 8} lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1555,7 +1560,7 @@ def test_truncates_full_line_because_of_max_chars(self) -> None: "a" * 10, "...", "", - "...Full output truncated (1 line hidden), use '-vv' to show", + "...Full output truncated, use '-vv' to show", ] def test_truncates_edgecase_when_truncation_message_makes_the_result_longer_for_chars( @@ -1580,7 +1585,6 @@ def test_truncates_at_8_lines_when_first_8_lines_are_EQ_max_chars(self) -> None: assert result != expl assert len(result) == 16 - 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "8 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1590,7 +1594,6 @@ def test_truncates_at_4_lines_when_first_4_lines_are_GT_max_chars(self) -> None: assert result != expl assert len(result) == 4 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "7 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1600,7 +1603,6 @@ def test_truncates_at_1_line_when_first_line_is_GT_max_chars(self) -> None: assert result != expl assert len(result) == 1 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "1000 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1608,7 +1610,6 @@ def test_full_output_truncated(self, monkeypatch, pytester: Pytester) -> None: """Test against full runpytest() output.""" line_count = 7 line_len = 100 - expected_truncated_lines = 2 pytester.makepyfile( rf""" def test_many_lines(): @@ -1627,7 +1628,7 @@ def test_many_lines(): [ "*+ 1*", "*+ 3*", - f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", + "*Full output truncated*use*-vv*", ] ) @@ -1641,7 +1642,7 @@ def test_many_lines(): [ "*+ 1*", "*+ 3*", - f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", + "*Full output truncated*use*-vv*", ] ) @@ -1697,9 +1698,7 @@ def test(): result = pytester.runpytest() if expected_lines_hidden != 0: - result.stdout.fnmatch_lines( - [f"*truncated ({expected_lines_hidden} lines hidden)*"] - ) + result.stdout.fnmatch_lines(["*Full output truncated*"]) else: result.stdout.no_fnmatch_line("*truncated*") result.stdout.fnmatch_lines( @@ -1710,6 +1709,92 @@ def test(): ) +class TestMaterializeWithTruncation: + """Tests for ``truncate.materialize_with_truncation``. + + Assertions check *behaviour* — that truncation kicks in / doesn't, + that the original lines are preserved, that the iterator's contract + is honoured — and never the literal footer wording. That way the + tests survive any future change to the truncation message format. + """ + + @staticmethod + def _config_with_limits(verbose: int = 0): + # Minimal stand-in for ``Config`` that ``materialize_with_truncation`` + # uses through ``_get_truncation_parameters``. + class C: + def getini(self, name: str) -> object: + return None # use defaults (8 lines / 640 chars) + + def get_verbosity(self, _verbosity_type: str | None = None) -> int: + return verbose + + return C() + + def test_iterator_within_limits_returns_all_lines(self) -> None: + lines = iter(["one", "two", "three"]) + result = truncate.materialize_with_truncation(lines, self._config_with_limits()) + assert result == ["one", "two", "three"] + + def test_iterator_exceeding_limits_is_truncated(self) -> None: + lines = (f"line {i}" for i in range(1000)) + result = truncate.materialize_with_truncation(lines, self._config_with_limits()) + # Bounded length — we kept the truncation footer plus at most a few + # lines past the cap; we never collect the full 1000-line stream. + assert len(result) < 20 + # The first lines we kept are the first lines of the input. + assert result[0] == "line 0" + # Some truncation marker is present (wording deliberately not asserted). + assert any("truncated" in line for line in result) + + def test_sized_input_returns_same_shape_as_iterator_input(self) -> None: + # When the input is already a sized container, the function still + # returns the truncated form; behaviour is the same as for an + # iterator over the same content. + content = [f"line {i}" for i in range(50)] + sized = truncate.materialize_with_truncation( + content, self._config_with_limits() + ) + unsized = truncate.materialize_with_truncation( + iter(content), self._config_with_limits() + ) + assert sized[0] == unsized[0] == "line 0" + assert any("truncated" in line for line in sized) + assert any("truncated" in line for line in unsized) + + def test_truncation_disabled_returns_full_input(self) -> None: + # verbose >= 2 disables truncation; the iterator is fully drained. + lines = (f"line {i}" for i in range(50)) + result = truncate.materialize_with_truncation( + lines, self._config_with_limits(verbose=2) + ) + assert result == [f"line {i}" for i in range(50)] + assert not any("truncated" in line for line in result) + + def test_first_lines_are_preserved_verbatim(self) -> None: + lines = (f"line {i}" for i in range(200)) + result = truncate.materialize_with_truncation(lines, self._config_with_limits()) + # The first kept lines should match the start of the input exactly + # (modulo the "..." appended to the last surviving line by the + # truncator, which we strip before comparing). + kept = [line.rstrip(".") for line in result if "truncated" not in line] + for i, line in enumerate(kept): + if line == "": + # Blank line separating content from the footer. + continue + assert line.startswith(f"line {i}") + + def test_idempotent_on_already_truncated_list(self) -> None: + # The dispatcher applies ``materialize_with_truncation`` after the + # built-in hook impl already truncated. Re-applying it must not + # corrupt the footer count or chop further lines. + once = truncate.materialize_with_truncation( + (f"line {i}" for i in range(200)), self._config_with_limits() + ) + twice = truncate.materialize_with_truncation(once, self._config_with_limits()) + assert twice == once + + def test_python25_compile_issue257(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2203,6 +2288,128 @@ def raise_exit(obj): callequal(1, 1) +def test_plugin_hook_returning_none_is_skipped(pytester: Pytester) -> None: + """A ``pytest_assertrepr_compare`` impl returning ``None`` is skipped + so the next impl (or the built-in) can produce the explanation. + Covers the ``if not new_expl: continue`` branch in ``callbinrepr``. + """ + pytester.makeconftest( + """ + def pytest_assertrepr_compare(op, left, right): + # Always defer to the next plugin / the built-in. + return None + """ + ) + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest() + # The built-in set-comparison explanation still reaches the user + # (so the None-returning hook did not swallow it). + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_plugin_hook_returning_empty_iterator_is_skipped(pytester: Pytester) -> None: + """A plugin returning a truthy but ultimately empty iterable is + skipped after materialisation. Covers the second + ``if not new_expl: continue`` branch in ``callbinrepr``. + """ + pytester.makeconftest( + """ + def pytest_assertrepr_compare(op, left, right): + # An iterator object is truthy, so it slips past the first + # falsy check; once materialised through truncation it is + # empty and the dispatcher must move on. + return iter([]) + """ + ) + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest() + # The built-in set-comparison explanation still reaches the user. + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_callbinrepr_falls_through_when_all_hooks_return_none( + pytester: Pytester, +) -> None: + """When every ``pytest_assertrepr_compare`` impl returns ``None`` + (no specialised explanation applies, e.g. ``assert 1 == 2``), the + dispatcher exhausts ``hook_result``, exits the loop, and returns + ``None``. Covers the ``continue → loop exit`` branch on the first + ``if not new_expl: continue`` line. + """ + pytester.makepyfile( + """ + def test_trivial(): + assert 1 == 2 + """ + ) + result = pytester.runpytest() + # Just the plain ``assert 1 == 2`` rewrite, with no specialised + # comparator explanation appended (because the dispatcher fell + # through to ``return None``). + result.stdout.fnmatch_lines(["*assert 1 == 2*"]) + result.assert_outcomes(failed=1) + + +def test_callbinrepr_plain_assert_mode(pytester: Pytester) -> None: + """In ``--assert=plain`` mode ``callbinrepr`` skips the ``%`` escape. + Covers the false branch of ``if item.config.getvalue("assertmode") + == "rewrite"``. + """ + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest("--assert=plain") + # In plain mode the comparator still runs via ``callbinrepr`` (it + # is the rewrite escaping that's skipped), so the explanation is + # still produced. + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_exception_before_first_yield_emits_summary_and_notice(monkeypatch) -> None: + """When the comparator raises *before* any explanation line has been + yielded, ``assertrepr_compare`` should still produce the summary so + the reader sees what was being compared, then append the failure + notice. Covers the ``summary_yielded is False`` branch of the + exception handler. + """ + from _pytest.assertion import _compare_any + + def raise_value_error(obj): + raise ValueError("synthetic repr failure") + + # ``istext`` is called inside ``_compare_eq_any`` before the first + # yield, so this triggers the failure path on the very first + # ``next()`` call from ``assertrepr_compare``. + monkeypatch.setattr(_compare_any, "istext", raise_value_error) + + expl = callequal(1, 1) + assert expl is not None + # Summary line still produced. + assert expl[0] == "1 == 1" + # The failure notice survives in the output; wording deliberately not + # asserted, only the underlying error's signature. + assert any("ValueError" in line or "synthetic" in line for line in expl) + + def test_assertion_location_with_coverage(pytester: Pytester) -> None: """This used to report the wrong location when run with coverage (#5754).""" p = pytester.makepyfile( @@ -2249,8 +2456,8 @@ def test(): """, [ "{bold}{red}E At index 1 diff: {reset}{number}1{hl-reset}{endline} != {reset}{number}2*", - "{bold}{red}E {light-red}- 2,{hl-reset}{endline}{reset}", - "{bold}{red}E {light-green}+ 1,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-red}- 2,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-green}+ 1,{hl-reset}{endline}{reset}", ], ), ( @@ -2268,8 +2475,8 @@ def test(): "{bold}{red}E Right contains 1 more item:{reset}", "{bold}{red}E {reset}{{{str}'{hl-reset}{str}number-is-0{hl-reset}{str}'{hl-reset}: {number}0*", "{bold}{red}E {reset}{light-gray} {hl-reset} {{{endline}{reset}", - "{bold}{red}E {light-gray} {hl-reset} 'number-is-1': 1,{endline}{reset}", - "{bold}{red}E {light-green}+ 'number-is-5': 5,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-gray} {hl-reset} 'number-is-1': 1,{endline}{reset}", + "{bold}{red}E {reset}{light-green}+ 'number-is-5': 5,{hl-reset}{endline}{reset}", ], ), (