Build/Test Tools: Add plugin compatibility testing workflow.

.github/workflows/reusable-plugin-compatibility.yml

@@ -0,0 +1,137 @@
+##
+# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be
+# activated against it without fataling.
+#
+# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then
+# the site is reset to the fresh install before the next one is installed. This keeps one broken plugin from
+# masking (or breaking) the next.
+#
+# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until
+# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the
+# site is checked with only the dependencies active first so that a dependency's own breakage is not reported
+# against the plugin under test.
+#
+# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that
+# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top
+# of that script.
+##
+name: Plugin Compatibility Tests
+
+on:
+  workflow_call:
+    inputs:
+      wp-version:
+        description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".'
+        required: false
+        type: 'string'
+        default: 'nightly'
+      php-version:
+        description: 'The version of PHP to use. Expected format: X.Y.'
+        required: false
+        type: 'string'
+        default: '8.5'
+      plugin-slugs:
+        description: 'A JSON array of WordPress.org plugin slugs to test in this shard. Ignored when fixtures is true.'
+        required: false
+        type: 'string'
+        default: '[]'
+      fixtures:
+        description: 'Test the fixture plugins in tools/plugin-compatibility/fixtures, which are built to break, and pass only when each gets the result expected for it.'
+        required: false
+        type: 'boolean'
+        default: false
+      db-type:
+        description: 'Database type. Valid types are mysql and mariadb.'
+        required: false
+        type: 'string'
+        default: 'mysql'
+      db-version:
+        description: 'Database version.'
+        required: false
+        type: 'string'
+        default: '9.7'
+
+# Disable permissions for all available scopes by default.
+# Any needed permissions should be configured at the job level.
+permissions: {}
+
+jobs:
+  # Tests that each plugin in the shard can be activated against the given version of WordPress.
+  #
+  # Performs the following steps:
+  # - Checks out the repository, for the script that runs the checks.
+  # - Sets up PHP and WP-CLI.
+  # - Runs the checks against every plugin in the shard.
+  plugin-compatibility-tests:
+    name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }}
+    permissions:
+      contents: read
+    runs-on: ubuntu-24.04
+    # Installing a plugin's dependencies before testing it adds a download and an activation for each
+    # one, and a dependency the size of WooCommerce is not quick to activate.
+    timeout-minutes: 45
+
+    services:
+      database:
+        # The database type and version are inputs so that this workflow can be pointed at any supported
+        # combination, which means the image cannot be pinned to a digest. This matches how the database
+        # service is declared in install-testing.yml and reusable-upgrade-testing.yml.
+        image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images]
+        ports:
+          - 3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval="30s"
+          --health-timeout="10s"
+          --health-retries="5"
+          -e MYSQL_ROOT_PASSWORD="root"
+          -e MYSQL_DATABASE="test_db"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
+          persist-credentials: false
+          sparse-checkout: tools/plugin-compatibility
+          sparse-checkout-cone-mode: false
+
+      - name: Set up PHP ${{ inputs.php-version }}
+        uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2
+        with:
+          php-version: '${{ inputs.php-version }}'
+          coverage: none
+          tools: wp-cli
+
+      # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the
+      # containers the script provisions when it is run from a terminal.
+      #
+      # WordPress is installed outside the workspace so that the checkout above is not part of the site under
+      # test.
+      - name: Test each plugin in isolation
+        env:
+          PLUGIN_SLUGS: ${{ inputs.plugin-slugs }}
+          FIXTURES: ${{ inputs.fixtures }}
+          WP_VERSION: ${{ inputs.wp-version }}
+          DB_PORT: ${{ job.services.database.ports['3306'] }}
+        run: |
+          set -uo pipefail
+
+          if [ "${FIXTURES}" = "true" ]; then
+            SELECTION=(
+              --fixtures=./tools/plugin-compatibility/fixtures
+              --expect=./tools/plugin-compatibility/fixtures/expected.txt
+            )
+          else
+            SELECTION=( --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" )
+          fi
+
+          ./tools/plugin-compatibility/test-plugins.sh \
+            --no-docker \
+            --wp-version="${WP_VERSION}" \
+            --dir="${RUNNER_TEMP}/wordpress" \
+            --db-host="127.0.0.1:${DB_PORT}" \
+            --db-name=test_db \
+            --db-user=root \
+            --db-pass=root \
+            "${SELECTION[@]}"

Gruntfile.js

@@ -98,10 +98,13 @@ module.exports = function(grunt) {
 			'.github/workflows/install-testing.yml',
 			'.github/workflows/test-and-zip-default-themes.yml',
 			'.github/workflows/install-testing.yml',
+			'.github/workflows/plugin-compatibility.yml',
 			'.github/workflows/slack-notifications.yml',
 			'.github/workflows/test-coverage.yml',
 			'.github/workflows/test-old-branches.yml',
-			'.github/workflows/upgrade-testing.yml'
+			'.github/workflows/upgrade-testing.yml',
+			// Tooling used only by the workflows above.
+			'tools/plugin-compatibility'
 		],
 
 		// Prepend `dir` to `file`, and keep `!` in place.

tools/plugin-compatibility/fixtures/expected.txt

@@ -0,0 +1,31 @@
+# The result each fixture plugin is expected to get from test-plugins.sh, in the order they are tested.
+#
+#     test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures --expect=tools/plugin-compatibility/fixtures/expected.txt
+#
+# Each fixture is a plugin built to break in one specific way, so that a change to the checks which stops a
+# broken plugin from being caught fails a run instead of quietly passing everything.
+#
+# Format: <slug> <PASS|FAIL|SKIPPED> [text the "Also active" or "Details" column must contain]
+
+fixture-healthy                    PASS
+fixture-fatal-on-load              FAIL     The plugin could not be activated
+fixture-fatal-on-activation        FAIL     The plugin could not be activated
+fixture-exit-on-activation         SKIPPED  the plugin was not active afterwards
+fixture-breaks-site-on-activation  FAIL     The request to / returned HTTP 500
+fixture-requires-newer-php         SKIPPED  declared requirements are not met
+fixture-requires-newer-wordpress   SKIPPED  declared requirements are not met
+fixture-requires-newer-both        SKIPPED  declared requirements are not met
+fixture-fatal-on-front-end         FAIL     The request to / returned HTTP 500
+fixture-fatal-on-login             FAIL     The request to /wp-login.php returned HTTP 500
+fixture-fatal-after-output         FAIL     PHP Fatal error
+fixture-parse-error-after-output   FAIL     PHP Parse error
+fixture-fatal-in-background-request FAIL    PHP Fatal error
+fixture-wp-cli-only-fatal          PASS     WP-CLI fatals when it loads WordPress
+fixture-requires-akismet           PASS     akismet
+fixture-requires-broken-dependency SKIPPED  The required plugins are not healthy on their own
+fixture-requires-wp-cli-only-fatal SKIPPED  WP-CLI cannot load WordPress with the required plugins active
+
+# These two must stay together and in this order: the second fatals if anything the first wrote survived the
+# reset between plugins.
+fixture-leaves-files-behind        PASS
+fixture-checks-for-leftovers       PASS

tools/plugin-compatibility/fixtures/fixture-breaks-site-on-activation/fixture-breaks-site-on-activation.php

@@ -0,0 +1,14 @@
+<?php
+/**
+ * Plugin Name: Fixture: Breaks the site on activation
+ * Description: Writes a must-use plugin that fatals when it is activated, so every request fails afterwards, even one that skips regular plugins. Expected to fail.
+ * Version: 1.0.0
+ */
+
+register_activation_hook(
+	__FILE__,
+	static function () {
+		wp_mkdir_p( WPMU_PLUGIN_DIR );
+		file_put_contents( WPMU_PLUGIN_DIR . '/fixture-broken-loader.php', "<?php\nfixture_function_that_does_not_exist();\n" );
+	}
+);

tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Broken dependency
+ * Description: Fatals on the front end. Only tested as a dependency of fixture-requires-broken-dependency.
+ * Version: 1.0.0
+ */
+
+add_action( 'template_redirect', 'fixture_function_that_does_not_exist' );

tools/plugin-compatibility/fixtures/fixture-checks-for-leftovers/fixture-checks-for-leftovers.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ * Plugin Name: Fixture: Checks for leftovers
+ * Description: Fatals if anything from fixture-leaves-files-behind survived the reset. Expected to pass when tested straight after it.
+ * Version: 1.0.0
+ */
+
+add_action(
+	'template_redirect',
+	static function () {
+		global $wpdb;
+
+		if ( file_exists( ABSPATH . 'fixture-leftover.txt' )
+			|| get_option( 'fixture_leftover' )
+			|| $wpdb->get_var( "SHOW TABLES LIKE '{$wpdb->prefix}fixture_leftover'" ) ) {
+			fixture_function_that_does_not_exist();
+		}
+	}
+);

tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php

@@ -0,0 +1,13 @@
+<?php
+/**
+ * Plugin Name: Fixture: Exit on activation
+ * Description: Exits from its activation hook, before core records it as active. Expected to be skipped rather than pass.
+ * Version: 1.0.0
+ */
+
+register_activation_hook(
+	__FILE__,
+	static function () {
+		exit;
+	}
+);

tools/plugin-compatibility/fixtures/fixture-fatal-after-output/fixture-fatal-after-output.php

@@ -0,0 +1,18 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal after output
+ * Description: Flushes the page and then fatals, so the response is still a 200. Expected to fail from the debug log.
+ * Version: 1.0.0
+ */
+
+add_action(
+	'wp_footer',
+	static function () {
+		while ( ob_get_level() ) {
+			ob_end_flush();
+		}
+		flush();
+
+		fixture_function_that_does_not_exist();
+	}
+);

tools/plugin-compatibility/fixtures/fixture-fatal-in-background-request/fixture-fatal-in-background-request.php

@@ -0,0 +1,28 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal in a background request
+ * Description: Starts a non-blocking request on the front page that fatals a few seconds later, after the page itself has returned. Expected to fail from the debug log.
+ * Version: 1.0.0
+ */
+
+add_action(
+	'template_redirect',
+	static function () {
+		wp_remote_post(
+			admin_url( 'admin-ajax.php?action=fixture_background_task' ),
+			array(
+				'blocking' => false,
+				'timeout'  => 0.01,
+			)
+		);
+	}
+);
+
+add_action(
+	'wp_ajax_nopriv_fixture_background_task',
+	static function () {
+		ignore_user_abort( true );
+		sleep( 3 );
+		fixture_function_that_does_not_exist();
+	}
+);

tools/plugin-compatibility/fixtures/fixture-fatal-on-activation/fixture-fatal-on-activation.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal on activation
+ * Description: Calls an undefined function from its activation hook. Expected to fail activation.
+ * Version: 1.0.0
+ */
+
+register_activation_hook( __FILE__, 'fixture_function_that_does_not_exist' );

tools/plugin-compatibility/fixtures/fixture-fatal-on-front-end/fixture-fatal-on-front-end.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal on the front end
+ * Description: Fatals before any output on front end requests. Expected to fail with an HTTP 500.
+ * Version: 1.0.0
+ */
+
+add_action( 'template_redirect', 'fixture_function_that_does_not_exist' );

tools/plugin-compatibility/fixtures/fixture-fatal-on-load/fixture-fatal-on-load.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal on load
+ * Description: Calls an undefined function as soon as it is loaded. Expected to fail activation.
+ * Version: 1.0.0
+ */
+
+fixture_function_that_does_not_exist();

tools/plugin-compatibility/fixtures/fixture-fatal-on-login/fixture-fatal-on-login.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Fatal on the login screen
+ * Description: Fatals before any output on the login screen only. Expected to fail with an HTTP 500.
+ * Version: 1.0.0
+ */
+
+add_action( 'login_init', 'fixture_function_that_does_not_exist' );

tools/plugin-compatibility/fixtures/fixture-healthy/fixture-healthy.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Healthy
+ * Description: Does nothing. Expected to pass.
+ * Version: 1.0.0
+ */
+
+add_action( 'wp_footer', '__return_null' );

tools/plugin-compatibility/fixtures/fixture-leaves-files-behind/fixture-leaves-files-behind.php

@@ -0,0 +1,21 @@
+<?php
+/**
+ * Plugin Name: Fixture: Leaves files behind
+ * Description: Writes a must-use plugin that loads this plugin, a root file, a table and an option. Expected to pass, and to leave nothing behind for the next plugin.
+ * Version: 1.0.0
+ */
+
+register_activation_hook(
+	__FILE__,
+	static function () {
+		global $wpdb;
+
+		// Fatals on every request once this plugin's directory is gone, unless the must-use plugin goes too.
+		wp_mkdir_p( WPMU_PLUGIN_DIR );
+		file_put_contents( WPMU_PLUGIN_DIR . '/fixture-leftover-loader.php', "<?php\nrequire WP_PLUGIN_DIR . '/fixture-leaves-files-behind/fixture-leaves-files-behind.php';\n" );
+		file_put_contents( ABSPATH . 'fixture-leftover.txt', 'left behind' );
+
+		$wpdb->query( "CREATE TABLE IF NOT EXISTS {$wpdb->prefix}fixture_leftover ( id INT )" );
+		update_option( 'fixture_leftover', 'left behind' );
+	}
+);

tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ * Plugin Name: Fixture: Parse error after output
+ * Description: Flushes the page and then includes a file with a syntax error. Expected to fail from the debug log.
+ * Version: 1.0.0
+ */
+
+add_action(
+	'wp_footer',
+	static function () {
+		while ( ob_get_level() ) {
+			ob_end_flush();
+		}
+		flush();
+
+		// Not a .php file, so that nothing linting the repository trips over it.
+		include __DIR__ . '/syntax-error.inc';
+	}
+);

tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/syntax-error.inc

@@ -0,0 +1,3 @@
+<?php
+// Deliberately invalid.
+function ( {

tools/plugin-compatibility/fixtures/fixture-requires-akismet/fixture-requires-akismet.php

@@ -0,0 +1,9 @@
+<?php
+/**
+ * Plugin Name: Fixture: Requires Akismet
+ * Description: Requires the Akismet plugin that ships with core, which is installed but not active. Expected to pass with Akismet active.
+ * Version: 1.0.0
+ * Requires Plugins: akismet
+ */
+
+add_action( 'wp_footer', '__return_null' );