perf(e2e): Run specs in parallel workers and overlap CI setup

.github/workflows/ci.yml

@@ -87,11 +87,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.2'
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
-          cache: 'npm'
 
       - name: Cache Composer
         uses: actions/cache@v4
@@ -101,32 +105,35 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-composer-unit-
 
-      # wp-env downloads WordPress and the core PHPUnit suite into this
-      # directory, which is roughly a gigabyte of git clones. Caching it keeps
-      # `env:start` from re-cloning them on every run. The key covers the
-      # wp-env config and version, because a change to either means different
-      # sources have to be downloaded.
-      - name: Cache wp-env sources
-        uses: actions/cache@v4
-        with:
-          path: ~/.wp-env
-          key: ${{ runner.os }}-wp-env-${{ hashFiles('.wp-env.json', 'package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-wp-env-
+      # This job only needs vendor/bin/phpunit (from Composer) and the wp-env
+      # CLI. A full `npm ci` pulls ~2,200 packages and has taken anywhere from
+      # 36s to 7 minutes on hosted runners; @wordpress/env alone is ~400
+      # packages and installs in ~30s. It is installed into a scratch prefix
+      # outside the repo so npm does not reconcile against package-lock.json
+      # and pull the whole tree anyway. The version is read from the lockfile
+      # so it cannot drift from what developers run locally.
+      - name: Install Composer dependencies
+        run: composer install --no-interaction --no-progress
 
-      # Also installs the Composer dependencies through the postinstall script.
-      - name: Install dependencies
-        run: npm ci
+      - name: Install wp-env
+        run: |
+          version=$(node -p "require('./package-lock.json').packages['node_modules/@wordpress/env'].version")
+          echo "Installing @wordpress/env@$version"
+          npm install --prefix "$RUNNER_TEMP/wp-env" --no-audit --no-fund "@wordpress/env@$version"
+          echo "$RUNNER_TEMP/wp-env/node_modules/.bin" >> "$GITHUB_PATH"
 
+      # The wp-env sources directory is deliberately not cached. A restored
+      # ~/.wp-env carries the previous run's install state, which skipped the
+      # plugin's activation hook and left the relationships table missing.
       - name: Start wp-env
-        run: npm run env:start
+        run: wp-env start
 
       - name: Run unit tests
-        run: npm run test:unit
+        run: wp-env run tests-cli --env-cwd="wp-content/plugins/$(basename "$PWD")" vendor/bin/phpunit
 
       - name: Stop wp-env
         if: always()
-        run: npm run env:stop
+        run: wp-env stop
 
   e2e:
     name: E2E (Playwright)
@@ -135,23 +142,78 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      # Not needed by Playwright, but every job that has this step completes
+      # `npm ci` in ~35s while this job, without it, took 2.5 to 4 minutes on
+      # the same runs (same npm, same cache hit). setup-php also installs
+      # Composer, which the postinstall hook calls.
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.2'
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
 
+      - name: Cache Composer
+        uses: actions/cache@v4
+        with:
+          path: ~/.composer/cache
+          key: ${{ runner.os }}-composer-e2e-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-e2e-
+
+      # Deliberately not `--ignore-scripts`: with the npm 10 that ships with
+      # the pinned Node, that flag made this step take 4 to 7 minutes instead
+      # of ~36s (a known npm 10 reify stall around lifecycle-script nodes).
       - name: Install dependencies
         run: npm ci
 
-      - name: Install Playwright browsers
-        run: npx playwright install --with-deps chromium
-
-      - name: Build assets
-        run: npm run build
+      - name: Get Playwright version
+        id: playwright-version
+        run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT"
 
-      - name: Start wp-env
-        run: npm run env:start
+      - name: Cache Playwright browsers
+        id: playwright-cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}
+
+      # wp-env spends most of its time pulling Docker images and installing
+      # WordPress, none of which depends on the Node-side steps. Run it in the
+      # background while the browser install and asset build proceed, then
+      # wait for it. All of this has to live in one step because `wait` only
+      # sees children of the same shell. wp-env output goes to a file and is
+      # printed afterwards so the interleaved log stays readable.
+      - name: Start wp-env, install browsers, build assets
+        env:
+          PLAYWRIGHT_CACHE_HIT: ${{ steps.playwright-cache.outputs.cache-hit }}
+        run: |
+          npm run env:start > wp-env-start.log 2>&1 &
+          wp_env_pid=$!
+
+          if [ "$PLAYWRIGHT_CACHE_HIT" = "true" ]; then
+            # Browser binaries came from cache; only the apt packages they
+            # need are missing on a fresh runner.
+            npx playwright install-deps chromium
+          else
+            npx playwright install --with-deps chromium
+          fi
+
+          npm run build
+
+          echo "::group::wp-env start"
+          if wait "$wp_env_pid"; then
+            cat wp-env-start.log
+            echo "::endgroup::"
+          else
+            cat wp-env-start.log
+            echo "::endgroup::"
+            exit 1
+          fi
 
       - name: Run E2E tests
         env:

.wp-env/mu-plugins/analytics-capture.php

@@ -11,23 +11,101 @@
  * proceed, which meant every local/CI test run was quietly leaking synthetic
  * events (and deactivation "feedback") into the real production collector.
  *
+ * Two additions support running the e2e suite in parallel Playwright
+ * workers against this single WordPress install: the capture log is
+ * per-worker (see cld_analytics_capture_worker_marker()), and Admin API
+ * calls made with the fake e2e credentials are answered locally (see
+ * cld_e2e_fake_cloud_intercept()).
+ *
  * @package Cloudinary
  */
 
 defined( 'ABSPATH' ) || exit;
 
 /**
- * Returns the path to the capture log file.
+ * Returns the e2e worker marker for the current request, if any.
+ *
+ * Playwright runs spec files in parallel workers against this single
+ * WordPress install. Each worker tags its browser/REST traffic with a
+ * `cld_e2e_worker` cookie and its WP-CLI calls with a `CLD_E2E_WORKER`
+ * env var, so every worker gets its own capture log and one worker's
+ * events (or `--clear`) can't leak into another worker's assertions.
+ *
+ * Requests without a marker (manual QA, fire-and-forget loopback threads
+ * spawned by the sync queue) fall back to the shared, unsuffixed log.
+ *
+ * @return string Sanitized marker, or empty string when none is present.
+ */
+function cld_analytics_capture_worker_marker() {
+	$marker = '';
+
+	// Dev/CI-only mu-plugin with no page cache in front of it, so the VIP
+	// cache-constraints sniff on $_COOKIE does not apply.
+	if ( ! empty( $_COOKIE['cld_e2e_worker'] ) ) { // phpcs:ignore WordPressVIPMinimum.Variables.RestrictedVariables.cache_constraints___COOKIE
+		$marker = wp_unslash( $_COOKIE['cld_e2e_worker'] ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized, WordPressVIPMinimum.Variables.RestrictedVariables.cache_constraints___COOKIE
+	} elseif ( false !== getenv( 'CLD_E2E_WORKER' ) && '' !== getenv( 'CLD_E2E_WORKER' ) ) {
+		$marker = getenv( 'CLD_E2E_WORKER' );
+	}
+
+	return preg_replace( '/[^A-Za-z0-9_-]/', '', (string) $marker );
+}
+
+/**
+ * Returns the path to the capture log file for the current worker.
  *
  * @return string
  */
 function cld_analytics_capture_log_path() {
 	$upload = wp_upload_dir();
+	$marker = cld_analytics_capture_worker_marker();
+	$suffix = '' !== $marker ? '-' . $marker : '';
 
-	return $upload['basedir'] . '/analytics-capture.log';
+	return $upload['basedir'] . '/analytics-capture' . $suffix . '.log';
 }
 
 add_filter( 'pre_http_request', 'cld_analytics_capture_intercept', 10, 3 );
+add_filter( 'pre_http_request', 'cld_e2e_fake_cloud_intercept', 10, 3 );
+
+/**
+ * Cloud name used by `fakeCloudinaryConnected()` in tests/e2e/utils/connection.js.
+ */
+const CLD_E2E_FAKE_CLOUD = 'e2e-fake-cloud';
+
+/**
+ * Short-circuits Cloudinary Admin API calls made with the fake e2e
+ * credentials.
+ *
+ * Analytics specs fake a connection so `Connect::is_connected()` is true.
+ * The dashboard then still calls the real Admin API for usage stats and
+ * per-day history (`Connect::history()` issues one request per day, and the
+ * 401 responses it gets are never cached because `is_wp_error()` entries
+ * are refetched). Each real round-trip is ~1s, so one `page=cloudinary`
+ * load can exceed Playwright's navigation timeout, and parallel workers
+ * multiply the load. Answer those calls locally with the same 401 the real
+ * API would return so the plugin's error handling still runs.
+ *
+ * @param false|array|WP_Error $preempt     Whether to preempt the request.
+ * @param array                $parsed_args Parsed request arguments.
+ * @param string               $url         The request URL.
+ *
+ * @return false|array|WP_Error
+ */
+function cld_e2e_fake_cloud_intercept( $preempt, $parsed_args, $url ) {
+	if ( false === strpos( $url, 'api.cloudinary.com/v1_1/' . CLD_E2E_FAKE_CLOUD . '/' ) ) {
+		return $preempt;
+	}
+
+	return array(
+		'headers'  => array( 'content-type' => 'application/json' ),
+		'body'     => wp_json_encode( array( 'error' => array( 'message' => 'Invalid credentials (e2e fake cloud)' ) ) ),
+		'response' => array(
+			'code'    => 401,
+			'message' => 'Unauthorized',
+		),
+		'cookies'  => array(),
+		'filename' => null,
+	);
+}
 
 /**
  * Logs outgoing analytics/deactivation-reason requests and preempts them

package.json

@@ -37,7 +37,9 @@
     "postinstall": "patch-package && composer install",
     "readme": "composer readme",
     "prepare": "husky",
-    "test:e2e": "playwright test --config tests/e2e/playwright.config.js",
+    "test:e2e": "npm-run-all --silent test:e2e:parallel test:e2e:serial",
+    "test:e2e:parallel": "playwright test --config tests/e2e/playwright.config.js --grep-invert @serial",
+    "test:e2e:serial": "playwright test --config tests/e2e/playwright.config.js --grep @serial --workers=1",
     "test:e2e:debug": "playwright test --config tests/e2e/playwright.config.js --ui",
     "test:unit": "wp-env run tests-cli --env-cwd=\"wp-content/plugins/$(basename \"$PWD\")\" vendor/bin/phpunit"
   },

tests/e2e/cache-analytics.spec.js

@@ -1,7 +1,7 @@
 /**
  * External dependencies
  */
-const { test, expect } = require( '@wordpress/e2e-test-utils-playwright' );
+const { test, expect } = require( './fixtures' );
 
 /**
  * Internal dependencies
@@ -53,8 +53,13 @@ test.describe( 'Non-media cache analytics', () => {
 		admin,
 		page,
 	} ) => {
-		createCachePoint();
+		// Load the admin page before creating the cache point. CACHE_POINT_PATH
+		// is not enabled in the cache settings, so an admin page load's
+		// Assets::activate_parents() treats an existing parent for it as
+		// disabled and deletes it. Creating the parent afterwards means the
+		// REST call below still finds it.
 		await admin.visitAdminPage( 'admin.php', 'page=cloudinary' );
+		createCachePoint();
 		const { restBase, nonce } = await getRestContext( page );
 
 		const response = await page.request.post( `${ restBase }/show_cache`, {
@@ -119,8 +124,24 @@ test.describe( 'Non-media cache analytics', () => {
 		// rather than relying on a subsequent admin page load's side effect
 		// (`Assets::update_asset_paths()`) to materialize it, which is a
 		// timing-sensitive path that has flaked under CI load.
+		//
+		// Also remove any leftover parent for CACHE_POINT_PATH (created by
+		// earlier tests in this file) and release the asset lock. That path
+		// is not enabled in settings, so the admin page load below would
+		// otherwise purge it via Assets::activate_parents() ->
+		// purge_parent() -> lock_assets(), a 10s transient nothing clears.
+		// While locked, get_assets_settings() returns nothing, no parent is
+		// activated, and rest_purge_all() never reaches the tracked branch.
+		// With sub-second page loads this test lands inside that window.
 		const realCachePoint = 'wp-content/uploads/';
 		wpEvalFile( `
+			$assets = get_plugin_instance()->get_component( 'assets' );
+			$stale  = $assets->get_asset_parent( '${ CACHE_POINT_PATH }' );
+			if ( $stale instanceof \\WP_Post ) {
+				wp_delete_post( $stale->ID, true );
+			}
+			$assets->unlock_assets();
+
 			$admin  = get_plugin_instance()->get_component( 'admin' );
 			$method = new \\ReflectionMethod( $admin, 'save_settings' );
 			$method->setAccessible( true );

tests/e2e/cloudinary-image-delivery.spec.js

@@ -3,7 +3,7 @@
  */
 const fs = require( 'fs' );
 const path = require( 'path' );
-const { test, expect } = require( '@wordpress/e2e-test-utils-playwright' );
+const { test, expect } = require( './fixtures' );
 
 /**
  * Internal dependencies
@@ -45,7 +45,9 @@ function expectCloudinaryUrl( rawUrl, expectedCloud ) {
 	).toBe( true );
 }
 
-test.describe( 'Cloudinary image delivery', () => {
+// @serial: needs real credentials in `cloudinary_connect` for `wp cloudinary
+// sync`, while every analytics spec overwrites that option with fake ones.
+test.describe( 'Cloudinary image delivery', { tag: '@serial' }, () => {
 	test.beforeAll( () => {
 		( { cloudName } = ensureCloudinaryConnected() );
 	} );

tests/e2e/cloudinary-video-delivery.spec.js

@@ -4,7 +4,7 @@
 const fs = require( 'fs' );
 const path = require( 'path' );
 const { execSync } = require( 'child_process' );
-const { test, expect } = require( '@wordpress/e2e-test-utils-playwright' );
+const { test, expect } = require( './fixtures' );
 
 /**
  * Internal dependencies
@@ -85,7 +85,9 @@ function setVideoPlayer( value ) {
 	} );
 }
 
-test.describe( 'Cloudinary video delivery', () => {
+// @serial: needs real credentials in `cloudinary_connect` for `wp cloudinary
+// sync`, while every analytics spec overwrites that option with fake ones.
+test.describe( 'Cloudinary video delivery', { tag: '@serial' }, () => {
 	test.beforeAll( () => {
 		( { cloudName } = ensureCloudinaryConnected() );
 	} );

tests/e2e/connection-analytics.spec.js

@@ -1,7 +1,7 @@
 /**
  * External dependencies
  */
-const { test, expect } = require( '@wordpress/e2e-test-utils-playwright' );
+const { test, expect } = require( './fixtures' );
 
 /**
  * Internal dependencies
@@ -29,7 +29,9 @@ const SEL = {
 	tab4: '#tab-4',
 };
 
-test.describe( 'Connection management analytics', () => {
+// @serial: resets and empties `cloudinary_connect`, disconnecting the plugin
+// for every other spec that happens to be running at the same time.
+test.describe( 'Connection management analytics', { tag: '@serial' }, () => {
 	test.beforeEach( async ( { context } ) => {
 		resetCloudinaryConnection();
 		clearAnalyticsEvents();