eslint · JoshuaKGoldberg · Apr 21, 2025 · Apr 22, 2025 · May 12, 2025 · May 12, 2025
@@ -0,0 +1,31 @@
+name: Test Ecosystem Plugins
+
+on:
+    push:
+        branches:
+            - "ecosystem/*"
+    schedule:
+        # “At 00:00.” https://crontab.guru/#0_0_*_*_*
+        - cron: "0 0 * * *"
+    workflow_dispatch: ~
+
+permissions: read-all
+
+jobs:
+    test_ecosystem:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v5
+            - uses: actions/setup-node@v6
+              with:
+                  node-version: "lts/*"
+            - run: npm install
+            - id: tester
+              run: npm run test:ecosystem -- --plugin all
+            - if: steps.tester.outcome == 'failure' && github.head_ref == 'main'
+              env:
+                  DISCORD_WEBHOOK: ${{ secrets.DISCORD_CONTRIBUTORS_WEBHOOK }}
+              uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9 # v0.3.2
+              with:
+                  args: "Ecosystem tests failed."
@@ -0,0 +1,25 @@
+name: Update Test Ecosystem Plugins
+
+on:
+    schedule:
+        # “At 00:00 on Monday.” https://crontab.guru/#0_0_*_*_1
+        - cron: "0 0 * * 1"
+    workflow_dispatch: ~
+
+permissions: read-all
+
+jobs:
+    test_ecosystem:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v5
+            - uses: actions/setup-node@v6
+              with:
+                  node-version: "lts/*"
+            - run: npm install
+            - run: npm run test:ecosystem:update
+            - uses: peter-evans/[email protected]
+              with:
+                  commit-message: "chore: update ecosystem plugins"
+                  title: "chore: update ecosystem plugins"
@@ -4,6 +4,7 @@
 test.js
 coverage/
 build/
+ecosystem/
 npm-debug.log
 yarn-error.log
 .pnpm-debug.log

@@ -25,6 +25,9 @@
         "tools/generate-formatter-examples.js",
       ],
       "ignoreDependencies": [
+        // Used in ecosystem.yml via cross-spawn shells
+        "@antfu/ni",
+        // Underlying code coverage engine used in unit tests
         "c8",
         // Optional peer dependency used for loading TypeScript configuration files
         "jiti",

@@ -70,6 +70,8 @@
     "test": "node Makefile.js test",
     "test:browser": "node Makefile.js cypress",
     "test:cli": "mocha",
+    "test:ecosystem": "node tools/test-ecosystem/index.mjs",
+    "test:ecosystem:update": "node tools/test-ecosystem/update.mjs",
     "test:emfile": "node tools/check-emfile-handling.js",
     "test:fuzz": "node Makefile.js fuzz",
     "test:performance": "node Makefile.js perf",
@@ -138,6 +140,7 @@
     "optionator": "^0.9.3"
   },
   "devDependencies": {
+    "@antfu/ni": "^28.0.0",
     "@arethetypeswrong/cli": "^0.18.0",
     "@babel/core": "^7.4.3",
     "@babel/preset-env": "^7.4.3",

@@ -0,0 +1,51 @@
+# Ecosystem Tests
+
+These tests run notable community plugins against the local ESLint repository.
+They're meant to validate that current changes to ESLint won't break downstream consumers.
+
+## Running
+
+To build and test all plugins:
+
+```shell
+npm run test:ecosystem
+```
+
+To run on just one plugin:
+
+```shell
+npm run test:ecosystem -- --plugin <plugin-name>
+```
+
+Plugins are stored in `plugins-data.json`.
+Plugin names are keys from that file.
+For example, to test against `@eslint/css`:
+
+```shell
+npm run test:ecosystem -- --plugin @eslint/css
+```
+
+### Debugging Commands
+
+The [`debug`](https://www.npmjs.com/package/debug) package is used to surface the stdout of commands when `DEBUG=test:ecosystem` is enabled.
+
+```shell
+DEBUG=test:ecosystem npm run test:ecosystem -- --plugin @eslint/css
+```
+
+## Updating
+
+`plugins-data.json` contains pinned commit hashes for each repository.
+Those hashes can be updated with the same script run in CI.
+
+To update all plugins:
+
+```shell
+npm run test:ecosystem:update
+```
+
+To update just one plugin:
+
+```shell
+npm run test:ecosystem:update -- --plugin <plugin-name>
+```
@@ -0,0 +1,84 @@
+/**
+ * @fileoverview Data utilities for ecosystem tests and updates to data.
+ * @author Josh Goldberg
+ */
+
+//-----------------------------------------------------------------------------
+// Requirements
+//-----------------------------------------------------------------------------
+
+import util, { styleText } from "node:util";
+
+//-----------------------------------------------------------------------------
+// Types
+//-----------------------------------------------------------------------------
+
+/**
+ * Command-line scripts to run on a plugin.
+ * @typedef {Object} PluginData
+ * @property {string?} build Command to build files before tests, if defined.
+ * @property {string} install Command to install dependencies.
+ * @property {string} test Command to run tests.
+ */
+
+/**
+ * Settings for how to clone, set up, and test an ecosystem plugin.
+ * @typedef {Object} PluginData
+ * @property {PluginCommands} commands Command-line scripts to run on the plugin.
+ * @property {string} commit Hash to check out after cloning the plugin.
+ * @property {string} repository Repository URL to clone the plugin from.
+ */
+
+//-----------------------------------------------------------------------------
+// Constants
+//-----------------------------------------------------------------------------
+
+export const pluginDataFilePath = new URL("plugins-data.json", import.meta.url);
+
+//-----------------------------------------------------------------------------
+// Functions
+//-----------------------------------------------------------------------------
+
+/**
+ * @param {"test" | "update"} action
+ * @returns {[string, PluginData][]}
+ */
+export async function getPlugins(action) {
+	const { values } = util.parseArgs({
+		options: {
+			plugin: {
+				type: "string",
+			},
+		},
+	});
+
+	const { plugin: pluginRequested = "all" } = values;
+	const { default: pluginsData } = await import(pluginDataFilePath, {
+		with: { type: "json" },
+	});
+
+	if (pluginRequested !== "all" && !(pluginRequested in pluginsData)) {
+		console.error(`The plugin "${values.plugin}" is not supported.`);
+		console.error(
+			`Supported plugins are: ${["", ...Object.keys(pluginsData)].join(
+				"\n  ",
+			)}`,
+		);
+		console.error(
+			`Alternately, run without --plugin to ${action} all plugins.`,
+		);
+		process.exit(1);
+	}
+
+	const pluginsSelected =
+		pluginRequested === "all"
+			? Object.entries(pluginsData)
+			: [[pluginRequested, pluginsData[pluginRequested]]];
+
+	console.log(
+		`Plugins to ${action}:`,
+		styleText("bold", pluginsSelected.map(([key]) => key).join(", ")),
+	);
+
+	return { pluginsData, pluginsSelected };
+}
@@ -0,0 +1,147 @@
+/**
+ * @fileoverview A utility to test ecosystem plugin(s) against the local ESLint.
+ * @author Josh Goldberg
+ */
+
+//-----------------------------------------------------------------------------
+// Requirements
+//-----------------------------------------------------------------------------
+
+import debug from "debug";
+import spawn from "cross-spawn";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { styleText } from "node:util";
+
+import { getPlugins } from "./data.mjs";
+
+const log = debug("test:ecosystem");
+
+/**
+ * @typedef {import("./data").PluginSettings} PluginSettings
+ */
+
+//-----------------------------------------------------------------------------
+// Helpers
+//-----------------------------------------------------------------------------
+
+/**
+ * Runs ecosystem tests for a single plugin. It will:
+ * 1. Clone the plugin repository into a sandbox directory
+ * 2. Check out the plugin's commit to test on
+ * 3. Install the plugin's dependencies
+ * 4. Link the local ESLint into the plugin
+ * 5. Build, if the plugin defines a build script
+ * 6. Run tests
+ * This intentionally does not try/catch: any errors will be thrown.
+ *
+ * @param {string} pluginKey
+ * @param {PluginSettings} pluginSettings
+ */
+async function runTests(pluginKey, pluginSettings) {
+	const directory = path.join(
+		SANDBOX_DIRECTORY,
+		pluginKey
+			.replaceAll(/[^a-z-]/g, " ")
+			.trim()
+			.replaceAll(" ", "-"),
+	);
+	console.log(styleText("bold", `Testing ${pluginKey} in ${directory}`));
+
+	/**
+	 * Attempts to run a command in the plugin sandbox directory.
+	 * If it fails, any error stdout will be logged in red before a re-throw.
+	 * @param {string} command
+	 * @param {string[]} args
+	 */
+	const runCommand = ([command, ...args]) => {
+		console.log(
+			styleText("gray", `[${pluginKey}] ${[command, ...args].join(" ")}`),
+		);
+		try {
+			return spawn.sync(command, args, {
+				cwd: directory,
+				stdio: log.enabled ? "inherit" : undefined,
+			});
+		} catch (error) {
+			console.error(
+				styleText("red", `[${pluginKey}]`),
+				error.stdout || error,
+			);
+			throw error;
+		}
+	};
+
+	// 1. Clone the plugin repository into a sandbox directory
+	await fs.mkdir(directory, { force: true });
+	runCommand([
+		"git",
+		"clone",
+		pluginSettings.repository,
+		directory,
+		"--depth",
+		"1",
+	]);
+
+	// 2. Check out the plugin's commit to test on
+	runCommand(["git", "fetch", "origin", pluginSettings.commit]);
+	runCommand(["git", "checkout", pluginSettings.commit]);
+
+	// 3. Install the plugin's dependencies
+	runCommand(["pwd"]);
+	runCommand(pluginSettings.commands.install);
+
+	// 4. Link the local ESLint into the plugin
+	runCommand(["npm", "link", "eslint"]);
+
+	// 5. Build, if the plugin defines a build script
+	if (pluginSettings.commands.build) {
+		runCommand(pluginSettings.commands.build);
+	}
+
+	// 6. Run the plugin's tests
+	runCommand(pluginSettings.commands.test);
+}
+
+//-----------------------------------------------------------------------------
+// Main
+//-----------------------------------------------------------------------------
+
+const { pluginsSelected } = await getPlugins("test");
+
+const SANDBOX_DIRECTORY = path.join(process.cwd(), "ecosystem");
+
+console.log(`Clearing existing sandbox directory: ${SANDBOX_DIRECTORY}`);
+await fs.rm(SANDBOX_DIRECTORY, {
+	force: true,
+	maxRetries: 8,
+	recursive: true,
+});
+await fs.mkdir(SANDBOX_DIRECTORY, { recursive: true });
+console.log("");
+
+const errors = [];
+
+// For each plugin to test, we try to runTests, recording thrown exceptions in errors
+for (const [pluginKey, pluginSettings] of pluginsSelected) {
+	try {
+		await runTests(pluginKey, pluginSettings);
+	} catch (error) {
+		errors.push({ error, pluginKey });
+	}
+
+	console.log("");
+}
+
+// If we had any errors, report them and exit as failed
+if (errors.length) {
+	console.error(styleText("red", "Errors occurred while testing plugins:"));
+	for (const { error, pluginKey } of errors) {
+		console.error(
+			`${styleText(["bold", "red"], pluginKey)}: ${styleText("red", `${error.stack || error}`)}`,
+		);
+	}
+	process.exitCode = 1;
+} else {
+	console.log(styleText("green", "All tests completed successfully."));
+}