Skip to content

Fail loudly when junit.xml is missing, empty, or unusable #339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ekamran wants to merge 1 commit into
base: master
Choose a base branch
from
+47 −7
Open

Fail loudly when junit.xml is missing, empty, or unusable #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions functions.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,12 @@ function trailingslashit( $str ) {
* - Number of errors.
* - Overall execution time.
*
* When the XML data is missing, empty, unparseable, or carries no usable
* result counts, execution stops with an error instead of returning an
* empty or hollow result, because such a result is either rejected much
* later with a confusing error or displayed in a misleading way. See
* issue #311.
*
* @param string $xml_string The JUnit XML data as a string. This should be
* well-formed XML representing the results of test executions, typically
* generated by testing frameworks compatible with JUnit reporting.
Expand All @@ -237,14 +243,41 @@ function trailingslashit( $str ) {
*/
function process_junit_xml( $xml_string ) {
if ( empty( $xml_string ) ) {
return '';
return '';
error_message( 'junit.xml is missing or empty, so there are no test results to report. Aborting instead of reporting an empty result as a success. See https://github.com/WordPress/phpunit-test-runner/issues/311.' );
}

$previous_libxml_setting = libxml_use_internal_errors( true );
libxml_clear_errors();

$xml = simplexml_load_string( $xml_string );

$errors = libxml_get_errors();
libxml_clear_errors();
libxml_use_internal_errors( $previous_libxml_setting );

if ( false === $xml ) {
$detail = '';
if ( ! empty( $errors ) ) {
$first = $errors[0];
$detail = ' First parser error: ' . trim( $first->message ) . ' at line ' . (int) $first->line . '.';
}
error_message( 'junit.xml could not be parsed as XML, so the test results are unreadable. Aborting instead of reporting an unreadable result as a success.' . $detail . ' See https://github.com/WordPress/phpunit-test-runner/issues/311.' );
}

$xml = simplexml_load_string( $xml_string );
$xml_string = null;
$project = $xml->testsuite;
$results = array();

// PHPUnit wraps everything in <testsuites>, but a root <testsuite>
// element is also a legitimate JUnit shape from other producers.
$project = 'testsuite' === $xml->getName() ? $xml : $xml->testsuite;
$results = array();

if (
! isset( $project['tests'] ) || '' === (string) $project['tests'] ||
! isset( $project['failures'] ) || '' === (string) $project['failures'] ||
! isset( $project['errors'] ) || '' === (string) $project['errors']
) {
error_message( 'junit.xml parsed but contains no usable test result counts, so the test results are unusable. A result without failure and error counts would be displayed as a success. Aborting instead. See https://github.com/WordPress/phpunit-test-runner/issues/311.' );
}

$results = array(
'tests' => (string) $project['tests'],
Expand All @@ -255,7 +288,7 @@ function process_junit_xml( $xml_string ) {

$results['testsuites'] = array();

$testsuites = $xml->xpath( '//testsuites//testsuite[ ( count( testcase ) > 0 ) and ( @errors > 0 or @failures > 0 ) ]' );
$testsuites = $xml->xpath( '//testsuite[ ( count( testcase ) > 0 ) and ( @errors > 0 or @failures > 0 ) ]' );
foreach ( $testsuites as $testsuite ) {
$result = array(
'name' => (string) $testsuite['name'],
Expand Down
9 changes: 8 additions & 1 deletion report.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,14 @@

// Process the junit.xml file.
log_message( 'Processing and uploading junit.xml' );
$xml = file_get_contents( $runner_vars['WPT_PREPARE_DIR'] . '/junit.xml' );
$junit_path = $runner_vars['WPT_PREPARE_DIR'] . '/junit.xml';
if ( ! file_exists( $junit_path ) ) {
error_message( 'junit.xml was not found at ' . $junit_path . ', so there are no test results to report. Aborting instead of reporting an empty result as a success. See https://github.com/WordPress/phpunit-test-runner/issues/311.' );
}
if ( ! is_readable( $junit_path ) ) {
error_message( 'junit.xml exists at ' . $junit_path . ' but is not readable. Fix the file permissions and run the report again. See https://github.com/WordPress/phpunit-test-runner/issues/311.' );
}
$xml = file_get_contents( $junit_path );
$results = process_junit_xml( $xml );

/*
Expand Down
Loading