From 776f82dc0653d7c96ed0c20c0d190fb8bea43047 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Wed, 15 Apr 2026 15:31:21 +0200 Subject: [PATCH 1/3] Route pytest.raises failures to FAIL instead of TEST_ERROR (#345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a test uses pytest.raises(SomeException) and the student's solution fails to raise that exception, pytest raises _pytest.outcomes.Failed. The ResultCollector hook only recognized AssertionError and pytest.fail.Exception, so the failure was routed to TestOutcome.TEST_ERROR and rendered as a misleading Syntax Error banner instead of a clean Failed row with the 'DID NOT RAISE …' message. Add _pytest.outcomes.Failed to the isinstance check so pytest.raises failures consistently surface as TestOutcome.FAIL across pytest versions. Import Failed explicitly to make the routing self-documenting. --- tutorial/tests/testsuite/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tutorial/tests/testsuite/helpers.py b/tutorial/tests/testsuite/helpers.py index b203a542..657a18f0 100644 --- a/tutorial/tests/testsuite/helpers.py +++ b/tutorial/tests/testsuite/helpers.py @@ -9,6 +9,7 @@ import ipywidgets import pytest +from _pytest.outcomes import Failed from IPython.display import Code from IPython.display import display as ipython_display from ipywidgets import HTML @@ -768,6 +769,7 @@ def pytest_exception_interact( TestOutcome.FAIL if exc.errisinstance(AssertionError) or exc.errisinstance(pytest.fail.Exception) + or exc.errisinstance(Failed) else TestOutcome.TEST_ERROR ) self.tests[report.nodeid] = TestCaseResult( From 1929d979e90242a7ae8c5f2da44072ae8dfc1ef4 Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Wed, 15 Apr 2026 15:32:15 +0200 Subject: [PATCH 2/3] Distinguish runtime errors from syntax errors in test result banner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestCaseResult.to_html() unconditionally rendered every TEST_ERROR outcome as '🚨 Syntax Error', even when the underlying exception was a plain AttributeError or TypeError raised from the student's solution code (e.g. 'NoneType' object has no attribute 'shape' when a stub is left as 'return'). The banner contradicted the actual exception type shown in the error block below. Branch the label on the exception class: real SyntaxError (including its IndentationError subclass) still renders as '🚨 Syntax Error' on the COMPILE_ERROR path; everything else renders as '⚠️ Runtime Error (ExcName)', reflecting what the student actually hit. --- tutorial/tests/testsuite/helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tutorial/tests/testsuite/helpers.py b/tutorial/tests/testsuite/helpers.py index 657a18f0..3b3b81d3 100644 --- a/tutorial/tests/testsuite/helpers.py +++ b/tutorial/tests/testsuite/helpers.py @@ -276,8 +276,15 @@ def to_html(self) -> str: status_text = "Failed" case TestOutcome.TEST_ERROR: status_class = "test-error" - icon = "🚨" - status_text = "Syntax Error" + if isinstance(self.exception, SyntaxError): + icon = "🚨" + status_text = "Syntax Error" + else: + icon = "⚠️" + exc_name = ( + type(self.exception).__name__ if self.exception else "Error" + ) + status_text = f"Runtime Error ({exc_name})" case _: status_class = "test-error" icon = "⚠️" From 62b6eb4941b8cb52d30994b3120f477d0436e6cc Mon Sep 17 00:00:00 2001 From: Edoardo Baldi Date: Wed, 15 Apr 2026 15:33:09 +0200 Subject: [PATCH 3/3] Preserve passing parametrizations when a sibling test errors run_pytest_for_function downgraded the entire result to IPytestOutcome.PYTEST_ERROR whenever any individual test had a TEST_ERROR outcome, discarding every TestCaseResult including parametrizations that had actually passed. The student saw a single generic error panel instead of per-case feedback. The IPytestOutcome.FINISHED display path already iterates test results and renders a mix of PASS/FAIL/TEST_ERROR correctly per-row, so the downgrade is redundant. Drop it. PYTEST_ERROR remains in use for pytest.ExitCode.INTERNAL_ERROR and for cell compilation failures wrapped by run_cell(). Drop the now-unused TestOutcome import from testsuite.py. --- tutorial/tests/testsuite/testsuite.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tutorial/tests/testsuite/testsuite.py b/tutorial/tests/testsuite/testsuite.py index 707643ad..79300d38 100644 --- a/tutorial/tests/testsuite/testsuite.py +++ b/tutorial/tests/testsuite/testsuite.py @@ -34,7 +34,6 @@ IPytestOutcome, IPytestResult, ResultCollector, - TestOutcome, TestResultOutput, ) @@ -66,20 +65,6 @@ def run_pytest_for_function( test_results=list(result_collector.tests.values()), ) case pytest.ExitCode.TESTS_FAILED: - if any( - test.outcome == TestOutcome.TEST_ERROR - for test in result_collector.tests.values() - ): - return IPytestResult( - function=function, - status=IPytestOutcome.PYTEST_ERROR, - exceptions=[ - test.exception - for test in result_collector.tests.values() - if test.exception - ], - ) - return IPytestResult( function=function, status=IPytestOutcome.FINISHED,