diff --git a/conformance/README.md b/conformance/README.md index 0b60a9490..7f28150dd 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -63,6 +63,8 @@ Test cases support the following special comments for declaring where errors sho * `# E[tag]`, where `tag` is an arbitrary string: must appear multiple times in a file with the same tag. Exactly one line with this tag must raise an error. * `# E[tag+]`: like `# E[tag]`, but errors may be raised on multiple lines. +* `# E[tag!]`: like `# E[tag]`, but zero or more lines with this tag may raise + an error while at least line must not raise an error. Each comment may be followed by a colon plus an explanation of the error; the explanation is ignored by the scoring system. diff --git a/conformance/src/main.py b/conformance/src/main.py index 63f82ea72..0073dcb07 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -6,9 +6,10 @@ import re import sys import tomllib +from collections.abc import Container, Sequence +from enum import Enum from pathlib import Path from time import time -from typing import Sequence import tomlkit @@ -63,17 +64,24 @@ def run_tests( update_type_checker_info(type_checker, root_dir) +class ErrorMultiplicity(Enum): + """How many lines with the same tag can have an error.""" + + SINGLE = 1 # exactly one line must have an error + MULTI = 2 # at least one line must have an error + REQUIRE_SUCCESS = 3 # at least one line must not have an error + + def get_expected_errors(test_case: Path) -> tuple[ dict[int, tuple[int, int]], - dict[str, tuple[list[int], bool]], + dict[str, tuple[list[int], ErrorMultiplicity]], ]: """Return the line numbers where type checkers are expected to produce an error. The return value is a tuple of two dictionaries: - The format of the first is {line number: (number of required errors, number of optional errors)}. - - The format of the second is {error tag: ([lines where the error may appear], allow multiple}. - If allow multiple is True, the error may appear on multiple lines; otherwise, it must - appear exactly once. + - The format of the second is {error tag: ([lines where the error may appear], error multiplicity)}. + See the ErrorMultiplicy enum for the second argument. For example, the following test case: @@ -92,7 +100,7 @@ def f(): pass # E[final] with open(test_case, "r", encoding="utf-8") as f: lines = f.readlines() output: dict[int, tuple[int, int]] = {} - groups: dict[str, tuple[list[int], bool]] = {} + groups: dict[str, tuple[list[int], ErrorMultiplicity]] = {} for i, line in enumerate(lines, start=1): line_without_comment, *_ = line.split("#") # Ignore lines with no non-comment content. This allows commenting out test cases. @@ -110,15 +118,18 @@ def f(): pass # E[final] for match in re.finditer(r"# E\[([^\]]+)\]", line): tag = match.group(1) if tag.endswith("+"): - allow_multiple = True + multiplicity = ErrorMultiplicity.MULTI + tag = tag[:-1] + elif tag.endswith("!"): + multiplicity = ErrorMultiplicity.REQUIRE_SUCCESS tag = tag[:-1] else: - allow_multiple = False + multiplicity = ErrorMultiplicity.SINGLE if tag not in groups: - groups[tag] = ([i], allow_multiple) + groups[tag] = ([i], multiplicity) else: - if groups[tag][1] != allow_multiple: - raise ValueError(f"Error group {tag} has inconsistent allow_multiple value in {test_case}") + if groups[tag][1] != multiplicity: + raise ValueError(f"Error group {tag} has inconsistent multiplicity value in {test_case}") groups[tag][0].append(i) for group, linenos in groups.items(): if len(linenos) == 1: @@ -151,21 +162,47 @@ def diff_expected_errors( differences.append(f"Line {expected_lineno}: Expected {expected_count} errors") # We don't report an issue if the count differs, because type checkers may produce # multiple error messages for a single line. - linenos_used_by_groups: set[int] = set() - for group, (linenos, allow_multiple) in error_groups.items(): - num_errors = sum(1 for lineno in linenos if lineno in errors) - if num_errors == 0: - differences.append(f"Lines {', '.join(map(str, linenos))}: Expected error (tag {group!r})") - elif num_errors == 1 or allow_multiple: - linenos_used_by_groups.update(linenos) - else: - differences.append(f"Lines {', '.join(map(str, linenos))}: Expected exactly one error (tag {group!r})") + + for group, (linenos, multiplicity) in error_groups.items(): + error = determine_group_error(group, linenos, multiplicity, errors) + if error is not None: + differences.append(error) + + linenos_used_by_groups = {ln for linenos, _ in error_groups.values() for ln in linenos} for actual_lineno, actual_errors in errors.items(): - if actual_lineno not in expected_errors and actual_lineno not in linenos_used_by_groups: - differences.append(f"Line {actual_lineno}: Unexpected errors {actual_errors}") + if ( + actual_lineno not in expected_errors + and actual_lineno not in linenos_used_by_groups + ): + differences.append( + f"Line {actual_lineno}: Unexpected errors {actual_errors}" + ) return "".join(f"{diff}\n" for diff in differences) +def determine_group_error( + group: str, + group_linenos: Sequence[int], + multiplicity: ErrorMultiplicity, + error_linenos: Container[int], +) -> str | None: + """Return the error message for the given group or None.""" + num_errors = sum(1 for lineno in group_linenos if lineno in error_linenos) + match multiplicity: + case ErrorMultiplicity.SINGLE: + if num_errors == 0: + return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})" + elif num_errors > 1: + return f"Lines {', '.join(map(str, group_linenos))}: Expected exactly one error (tag {group!r})" + case ErrorMultiplicity.MULTI: + if num_errors == 0: + return f"Lines {', '.join(map(str, group_linenos))}: Expected error (tag {group!r})" + case ErrorMultiplicity.REQUIRE_SUCCESS: + if num_errors == len(group_linenos): + return f"Lines {', '.join(map(str, group_linenos))}: Expected at least one success (tag {group!r})" + return None + + def update_output_for_test( type_checker: TypeChecker, results_dir: Path,