diff --git a/changelog/4569.bugfix.rst b/changelog/4569.bugfix.rst new file mode 100644 index 00000000000..97968182058 --- /dev/null +++ b/changelog/4569.bugfix.rst @@ -0,0 +1 @@ +:option:`-k` no longer matches attributes that end up in a test function's ``__dict__`` without being meant as keywords, such as pytest's own ``pytestmark`` attribute and the bookkeeping left behind by :func:`functools.wraps` and :func:`functools.lru_cache`. diff --git a/changelog/4569.doc.rst b/changelog/4569.doc.rst new file mode 100644 index 00000000000..e5dc171983d --- /dev/null +++ b/changelog/4569.doc.rst @@ -0,0 +1 @@ +``pytest --help``, the :option:`-k` reference entry and :ref:`select-tests` no longer describe :option:`-k` as a Python expression matched against test names only; :ref:`keyword expressions` now shows marker names being matched as well. The examples that looked up a marker use :meth:`Node.get_closest_marker <_pytest.nodes.Node.get_closest_marker>` rather than ``item.keywords``. diff --git a/changelog/4569.improvement.rst b/changelog/4569.improvement.rst new file mode 100644 index 00000000000..c4bff2f36d3 --- /dev/null +++ b/changelog/4569.improvement.rst @@ -0,0 +1 @@ +:meth:`Node.add_marker <_pytest.nodes.Node.add_marker>` now stores a :class:`~pytest.Mark` in ``node.keywords``, matching what marks applied during collection store. Previously it stored a :class:`~pytest.MarkDecorator`. diff --git a/doc/en/example/markers.rst b/doc/en/example/markers.rst index c8e4172a696..d88c2c7c159 100644 --- a/doc/en/example/markers.rst +++ b/doc/en/example/markers.rst @@ -162,15 +162,16 @@ Or select multiple nodes: when running pytest with the ``-rf`` option. You can also construct Node IDs from the output of ``pytest --collect-only``. -Using ``-k expr`` to select tests based on their name +.. _`keyword expressions`: + +Using ``-k expr`` to select tests by keyword ------------------------------------------------------- .. versionadded:: 2.0/2.3.4 You can use the :option:`-k` command line option to specify an expression -which implements a substring match on the test names instead of the -exact match on markers that :option:`-m` provides. This makes it easy to -select tests based on their names: +which implements a substring match on the test's *keywords*, instead of the +exact match on markers that :option:`-m` provides. .. versionchanged:: 5.4 @@ -224,10 +225,37 @@ Or to select "http" and "quick" tests: You can use ``and``, ``or``, ``not`` and parentheses. +A test's own name is only one of its keywords. The others are: + +* the names of the test's parents, usually the file and class it is in + (``test_server.py``, ``TestClass``); +* the names of the markers applied to it or to its parents (``webtest``, + ``device``); +* attributes set on the test function, as in the legacy ``test_func.slow = True`` + style; +* any :attr:`extra keywords <_pytest.nodes.Node.extra_keyword_matches>` + explicitly added to it or to its parents. + +Marker names being keywords is why ``-k webtest`` selects ``test_send_http``, +whose name contains no ``webtest`` at all: + +.. code-block:: pytest + + $ pytest -v -k webtest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 3 deselected / 1 selected + + test_server.py::test_send_http PASSED [100%] + + ===================== 1 passed, 3 deselected in 0.12s ====================== -In addition to the test's name, :option:`-k` also matches the names of the test's parents (usually, the name of the file and class it's in), -attributes set on the test function, markers applied to it or its parents and any :attr:`extra keywords <_pytest.nodes.Node.extra_keyword_matches>` -explicitly added to it or its parents. +A marker's arguments are not keywords, though, and the match is a substring +one: ``-k device`` selects both ``device`` tests, where +:ref:`the -m expression above ` selects only +the one. Use :option:`-m` when you want markers and nothing else. Registering markers diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index dff53488c88..090a81575fa 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -274,7 +274,7 @@ line option to control skipping of ``pytest.mark.slow`` marked tests: return skip_slow = pytest.mark.skip(reason="need --runslow option to run") for item in items: - if "slow" in item.keywords: + if item.get_closest_marker("slow"): item.add_marker(skip_slow) We can now write a test module like this: @@ -560,7 +560,7 @@ an ``incremental`` marker which is to be used on classes: def pytest_runtest_makereport(item, call): - if "incremental" in item.keywords: + if item.get_closest_marker("incremental"): # incremental marker is used if call.excinfo is not None: # the test has failed @@ -581,7 +581,7 @@ an ``incremental`` marker which is to be used on classes: def pytest_runtest_setup(item): - if "incremental" in item.keywords: + if item.get_closest_marker("incremental"): # retrieve the class name of the test cls_name = str(item.cls) # check if a previous test has failed for this class diff --git a/doc/en/how-to/usage.rst b/doc/en/how-to/usage.rst index 35b07bfe8c1..91f48ffe6fa 100644 --- a/doc/en/how-to/usage.rst +++ b/doc/en/how-to/usage.rst @@ -38,8 +38,9 @@ Pytest supports several ways to run and select tests from the command-line or fr pytest -k 'MyClass and not method' -This will run tests which contain names that match the given *string expression* (case-insensitive), -which can include Python operators that use filenames, class names and function names as variables. +This will run tests whose *keywords* match the given expression (case-insensitive). +A test's keywords are its own name, the names of the file and class it is in, the names of +its markers, and :ref:`a few more `. The example above will run ``TestMyClass.test_something`` but not ``TestMyClass.test_method_simple``. Use ``""`` instead of ``''`` in expression when running this on Windows diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 63c673547ef..54074262401 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -2870,19 +2870,26 @@ Test Selection .. option:: -k EXPRESSION - Only run tests which match the given substring expression. - An expression is a Python evaluable expression where all names are substring-matched against test names and their parent classes. + Only run tests which match the given keyword expression. + An expression is made of names combined with ``and``, ``or``, ``not`` and parentheses. + Each name is matched case-insensitively as a substring of any of the test's keywords. Examples:: - pytest -k "test_method or test_other" # matches names containing 'test_method' OR 'test_other' - pytest -k "not test_method" # matches names NOT containing 'test_method' + pytest -k "test_method or test_other" # matches keywords containing 'test_method' OR 'test_other' + pytest -k "not test_method" # matches keywords NOT containing 'test_method' pytest -k "not test_method and not test_other" # excludes both - The matching is case-insensitive. - Keywords are also matched to classes and functions containing extra names in their ``extra_keyword_matches`` set. + The keywords of a test are: - See :ref:`select-tests` for more information and examples. + * its own name, including any parametrization id; + * the names of its parent class, module and directories; + * the names of the markers applied to it or to any of its parents, as bare names: + unlike :option:`-m`, ``-k`` matches them as substrings and cannot match their arguments; + * attributes assigned directly to the test function, as in the legacy ``test_func.slow = True`` style; + * any names added to the :attr:`~_pytest.nodes.Node.extra_keyword_matches` set of it or of a parent. + + See :ref:`keyword expressions` for more information and examples. .. option:: -m MARKEXPR @@ -2891,9 +2898,12 @@ Test Selection Examples:: - pytest -m slow # run tests marked with @pytest.mark.slow - pytest -m "not slow" # run tests NOT marked slow - pytest -m "mark1 and not mark2" # run tests marked mark1 but not mark2 + pytest -m slow # run tests marked with @pytest.mark.slow + pytest -m "not slow" # run tests NOT marked slow + pytest -m "mark1 and not mark2" # run tests marked mark1 but not mark2 + pytest -m "device(serial='123')" # run tests marked device with that argument + + Marker names are matched exactly and case-sensitively. See :ref:`mark` for more information on markers. @@ -3441,21 +3451,19 @@ All the command-line flags can also be obtained by running ``pytest --help``:: file_or_dir general: - -k EXPRESSION Only run tests which match the given substring - expression. An expression is a Python evaluable - expression where all names are substring-matched - against test names and their parent classes. - Example: -k 'test_method or test_other' matches all - test functions and classes whose name contains - 'test_method' or 'test_other', while -k 'not - test_method' matches those that don't contain - 'test_method' in their names. -k 'not test_method - and not test_other' will eliminate the matches. - Additionally keywords are matched to classes and - functions containing extra names in their - 'extra_keyword_matches' set, as well as functions - which have names assigned directly to them. The - matching is case-insensitive. + -k EXPRESSION Only run tests matching the given keyword expression, + e.g. -k 'test_method or test_other', -k 'not (slow or + network)'. + Names in the expression are matched case-insensitively + as substrings of the test's keywords, which are: + - its own name, including any parametrization id + - the names of its class, module and directories + - the names of the markers applied to it or to its + parents + - attributes assigned directly to the test function + - any names in its 'extra_keyword_matches' set + Use -m to match marker names exactly, including their + arguments. -m MARKEXPR Only run tests matching given mark expression. For example: -m 'mark1 and not mark2'. --markers show markers (builtin, plugin and per-project ones). diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 7656fca2f5b..40366065d82 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -618,7 +618,7 @@ def path(self) -> Path: @property def keywords(self) -> MutableMapping[str, Any]: - """Keywords/markers dictionary for the underlying node.""" + """The :attr:`~_pytest.nodes.Node.keywords` of the underlying node.""" node: nodes.Node = self.node return node.keywords @@ -634,10 +634,11 @@ def addfinalizer(self, finalizer: Callable[[], object]) -> None: raise NotImplementedError() def applymarker(self, marker: str | MarkDecorator) -> None: - """Apply a marker to a single test function invocation. + """Apply a marker to the test(s) this fixture is running for. - This method is useful if you don't want to have a keyword/marker - on all function invocations. + Unlike decorating a test function, which marks every invocation of it, + this adds the marker to the request's ``node``: for a function-scoped + fixture, the single test invocation currently being set up. :param marker: An object created by a call to ``pytest.mark.NAME(...)``. diff --git a/src/_pytest/mark/__init__.py b/src/_pytest/mark/__init__.py index 73354506df3..996322d93f1 100644 --- a/src/_pytest/mark/__init__.py +++ b/src/_pytest/mark/__init__.py @@ -94,18 +94,16 @@ def pytest_addoption(parser: Parser) -> None: dest="keyword", default="", metavar="EXPRESSION", - help="Only run tests which match the given substring expression. " - "An expression is a Python evaluable expression " - "where all names are substring-matched against test names " - "and their parent classes. Example: -k 'test_method or test_" - "other' matches all test functions and classes whose name " - "contains 'test_method' or 'test_other', while -k 'not test_method' " - "matches those that don't contain 'test_method' in their names. " - "-k 'not test_method and not test_other' will eliminate the matches. " - "Additionally keywords are matched to classes and functions " - "containing extra names in their 'extra_keyword_matches' set, " - "as well as functions which have names assigned directly to them. " - "The matching is case-insensitive.", + help="Only run tests matching the given keyword expression, " + "e.g. -k 'test_method or test_other', -k 'not (slow or network)'.\n" + "Names in the expression are matched case-insensitively as substrings " + "of the test's keywords, which are:\n" + "- its own name, including any parametrization id\n" + "- the names of its class, module and directories\n" + "- the names of the markers applied to it or to its parents\n" + "- attributes assigned directly to the test function\n" + "- any names in its 'extra_keyword_matches' set\n" + "Use -m to match marker names exactly, including their arguments.", ) group._addoption( # private to use reserved lower-case short option @@ -150,19 +148,33 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None: return None +#: Attributes which are never meaningful as keywords, but do end up in the +#: ``__dict__`` of a test function: pytest's own mark storage, and the +#: bookkeeping decorators leave behind (``functools.wraps`` copies ``__wrapped__`` +#: and friends, ``functools.lru_cache`` adds ``cache_parameters``, ...). +IGNORED_FUNCTION_ATTRIBUTES = frozenset({"pytestmark", "cache_parameters"}) + + +def _is_matchable_function_attribute(name: str) -> bool: + """Whether a test function attribute may be matched by ``-k``.""" + return not name.startswith("_") and name not in IGNORED_FUNCTION_ATTRIBUTES + + @dataclasses.dataclass class KeywordMatcher: - """A matcher for keywords. + """A matcher for keywords, as used by ``-k``. - Given a list of names, matches any substring of one of these names. The + Given a set of names, matches any substring of one of these names. The string inclusion check is case-insensitive. - Will match on the name of colitem, including the names of its parents. - Only matches names of items which are either a :class:`Class` or a - :class:`Function`. + The names are collected in :meth:`from_item` from the item and its + parents: their node names, the names of the markers in scope, the + attributes assigned to the test function, and the + :attr:`~_pytest.nodes.Node.extra_keyword_matches` sets. - Additionally, matches on names in the 'extra_keyword_matches' set of - any item, as well as names directly assigned to test functions. + Note that these names are collected independently of + :attr:`Node.keywords <_pytest.nodes.Node.keywords>`; writing into that + mapping does not affect ``-k``. """ __slots__ = ("_names",) @@ -190,10 +202,16 @@ def from_item(cls, item: Item) -> KeywordMatcher: # Add the names added as extra keywords to current or parent items. mapped_names.update(item.listextrakeywords()) - # Add the names attached to the current function through direct assignment. + # Add the names attached to the current function through direct + # assignment, ignoring the attributes that merely happen to live in the + # function's __dict__ without anyone meaning them as keywords. function_obj = getattr(item, "function", None) if function_obj: - mapped_names.update(function_obj.__dict__) + mapped_names.update( + name + for name in function_obj.__dict__ + if _is_matchable_function_attribute(name) + ) # Add the markers to the keywords as we no longer handle them correctly. mapped_names.update(mark.name for mark in item.iter_markers()) diff --git a/src/_pytest/nodes.py b/src/_pytest/nodes.py index 76d480348de..2b510907122 100644 --- a/src/_pytest/nodes.py +++ b/src/_pytest/nodes.py @@ -185,13 +185,21 @@ def __init__( self.path: pathlib.Path = path # The explicit annotation is to avoid publicly exposing NodeKeywords. - #: Keywords/markers collected from all scopes. + #: Mapping of the names collected for this node and its parents: the + #: node names themselves, the names of the markers applied to them + #: (mapping to the :class:`~pytest.Mark`), and, for a test function, + #: its attributes and parametrization id. + #: + #: Mostly useful for ``"markname" in item.keywords`` checks. Note that + #: this mapping is not what ``-k`` matches against, so writing to it + #: does not affect test selection, and that it is unrelated to + #: :attr:`extra_keyword_matches`. self.keywords: MutableMapping[str, Any] = NodeKeywords(self) #: The marker objects belonging to this node. self.own_markers: list[Mark] = [] - #: Allow adding of extra keywords to use for matching. + #: Extra names for ``-k`` to match this node and its children on. self.extra_keyword_matches: set[str] = set() if nodeid is not None: @@ -335,7 +343,7 @@ def add_marker(self, marker: str | MarkDecorator, append: bool = True) -> None: marker_ = getattr(MARK_GEN, marker) else: raise ValueError("is not a string or pytest.mark.* Marker") - self.keywords[marker_.name] = marker_ + self.keywords[marker_.name] = marker_.mark if append: self.own_markers.append(marker_.mark) else: diff --git a/src/_pytest/python.py b/src/_pytest/python.py index bc3d55243f7..792d0d5b4f3 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -1670,7 +1670,8 @@ class Function(PyobjMixin, nodes.Item): If given, the object which will be called when the Function is invoked, otherwise the callobj will be obtained from ``parent`` using ``originalname``. :param keywords: - Keywords bound to the function object for "-k" matching. + Extra entries for :attr:`~_pytest.nodes.Node.keywords`, taking + precedence over the function's attributes and markers. :param session: The pytest Session object. :param fixtureinfo: diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 35a212e8eab..722f71909d4 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -345,7 +345,7 @@ def __init__( self, nodeid: str | NodeId, location: tuple[str, int | None, str], - keywords: Mapping[str, Any], + keywords: Mapping[str, Literal[1]], outcome: Literal["passed", "failed", "skipped"], longrepr: ExceptionInfo[BaseException] | tuple[str, int, str] @@ -370,9 +370,10 @@ def __init__( #: The line number is 0-based. self.location: tuple[str, int | None, str] = location - #: A name -> value dictionary containing all keywords and - #: markers associated with a test invocation. - self.keywords: Mapping[str, Any] = keywords + #: The names in :attr:`Node.keywords <_pytest.nodes.Node.keywords>` + #: of the item, each mapping to ``1``: only the names survive into the + #: report, the values of the node keywords are not carried over. + self.keywords: Mapping[str, Literal[1]] = keywords #: Test outcome, always one of "passed", "failed", "skipped". self.outcome = outcome @@ -419,7 +420,7 @@ def from_item_and_call(cls, item: Item, call: CallInfo[None]) -> TestReport: duration = call.duration start = call.start stop = call.stop - keywords = {x: 1 for x in item.keywords} + keywords: Mapping[str, Literal[1]] = dict.fromkeys(item.keywords, 1) excinfo = call.excinfo sections = [] if not call.excinfo: diff --git a/testing/test_collection.py b/testing/test_collection.py index 093162ddec4..f7cdc60a1c3 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -973,6 +973,16 @@ def test_method(self): pass assert "bar" not in mod.keywords assert "baz" not in mod.keywords + def test_added_marks_added_to_keywords(self, pytester: Pytester) -> None: + """Dynamically added marks land in keywords as Mark objects, same as + marks applied during collection (#4569).""" + item = pytester.getitem("def test_method(): pass", "test_method") + item.add_marker("foo") + item.add_marker(pytest.mark.bar("arg", kwarg=1)) + + assert item.keywords["foo"] == pytest.mark.foo.mark + assert item.keywords["bar"] == pytest.mark.bar("arg", kwarg=1).mark + class TestCollectDirectoryHook: def test_custom_directory_example(self, pytester: Pytester) -> None: diff --git a/testing/test_mark.py b/testing/test_mark.py index 9698aca35a7..82a941df0d6 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -1057,6 +1057,36 @@ def test_one(): _passed, _skipped, failed = reprec.countoutcomes() assert failed == 1 + @pytest.mark.parametrize("keyword", ["pytestmark", "wrapped", "cache_parameters"]) + def test_no_match_on_ignored_function_attributes( + self, pytester: Pytester, keyword: str + ) -> None: + """`-k` ignores attributes that end up in a test function's __dict__ + without being meant as keywords (#4569).""" + pytester.makepyfile( + """ + import functools + import pytest + + def deco(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + return fn(*args, **kwargs) + return wrapper + + @pytest.mark.some_mark + def test_marked(): pass + + @deco + def test_decorated(): pass + + @functools.lru_cache + def test_cached(): pass + """ + ) + result = pytester.runpytest("-k", keyword) + result.assert_outcomes(deselected=3) + @pytest.mark.xfail def test_keyword_extra_dash(self, pytester: Pytester) -> None: p = pytester.makepyfile( @@ -1128,6 +1158,106 @@ def get_collected_names(*args: str) -> list[str]: # do not collect anything based on names outside the collection tree assert get_collected_names("-k", pytester._name) == [] + def test_keyword_matches_marks_from_parents(self, pytester: Pytester) -> None: + """`-k` matches marker names from the module, class and base classes.""" + pytester.makepyfile( + """ + import pytest + pytestmark = pytest.mark.modmark + + @pytest.mark.basemark + class Base: + def test_inherited(self): pass + + @pytest.mark.classmark + class TestClass(Base): + def test_method(self): pass + + def test_toplevel(): pass + """ + ) + for keyword, selected in [ + ("modmark", 3), + ("classmark", 2), + ("basemark", 2), + ]: + result = pytester.runpytest("-k", keyword) + result.assert_outcomes(passed=selected, deselected=3 - selected) + + def test_keyword_does_not_match_mark_arguments(self, pytester: Pytester) -> None: + """`-k` matches marker names, not what the marker was called with.""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.mymark("someargument", somekwarg="anotherargument") + def test_one(): pass + """ + ) + for keyword in ["someargument", "anotherargument", "somekwarg"]: + result = pytester.runpytest("-k", keyword) + result.assert_outcomes(deselected=1) + + def test_keyword_matches_dynamically_added_mark(self, pytester: Pytester) -> None: + """A mark added in a conftest `pytest_collection_modifyitems` is seen by + `-k`, because conftest hook implementations run before the ones of the + mark plugin doing the deselection.""" + pytester.makeconftest( + """ + def pytest_collection_modifyitems(items): + for item in items: + if item.name == "test_one": + item.add_marker("addedmark") + """ + ) + pytester.makepyfile( + """ + def test_one(): pass + def test_two(): pass + """ + ) + result = pytester.runpytest("-k", "addedmark") + result.assert_outcomes(passed=1, deselected=1) + + def test_keyword_matches_item_extra_keyword_matches( + self, pytester: Pytester + ) -> None: + """`extra_keyword_matches` is honoured on the item itself, not just on + a parent collector.""" + pytester.makeconftest( + """ + def pytest_collection_modifyitems(items): + for item in items: + if item.name == "test_one": + item.extra_keyword_matches.add("extrakeyword") + """ + ) + pytester.makepyfile( + """ + def test_one(): pass + def test_two(): pass + """ + ) + result = pytester.runpytest("-k", "extrakeyword") + result.assert_outcomes(passed=1, deselected=1) + + @pytest.mark.parametrize("option", ["-k", "-m"]) + def test_writing_to_keywords_does_not_select( + self, pytester: Pytester, option: str + ) -> None: + """Writing into `item.keywords` does not make a test selectable: `-k` + collects its names separately, and `-m` only looks at markers.""" + pytester.makeconftest( + """ + def pytest_collection_modifyitems(items): + for item in items: + item.keywords["setviakeywords"] = True + """ + ) + pytester.makepyfile("def test_one(): pass") + result = pytester.runpytest(option, "setviakeywords") + result.assert_outcomes(deselected=1) + class TestMarkDecorator: @pytest.mark.parametrize( @@ -1287,12 +1417,16 @@ class TestBarClass(BaseTests): deselected_tests = dlist[0].items assert len(deselected_tests) == 1 - # todo: fixed - # keywords smear - expected behaviour - # reprec_keywords = pytester.inline_run("-k", "FOO") - # passed_k, skipped_k, failed_k = reprec_keywords.countoutcomes() - # assert passed_k == 2 - # assert skipped_k == failed_k == 0 + # Marks used to smear onto the shared base class function object, so that + # -k FOO matched both subclasses; it no longer does. + reprec_keywords = pytester.inline_run("-k", "FOO") + passed_k, skipped_k, failed_k = reprec_keywords.countoutcomes() + assert passed_k == 1 + assert skipped_k == failed_k == 0 + + # -m matches marker names exactly, so the case has to match too. + reprec_lower = pytester.inline_run("-m", "foo") + assert reprec_lower.countoutcomes() == [0, 0, 0] def test_addmarker_order(pytester) -> None: