eslint · JoshuaKGoldberg · Apr 21, 2025 · Apr 22, 2025 · May 12, 2025 · May 12, 2025
@@ -0,0 +1,31 @@
+name: Test Ecosystem Plugins
+
+on:
+    push:
+        branches:
+            - "ecosystem/*"
+    schedule:
+        # “At 00:00.” https://crontab.guru/#0_0_*_*_*
+        - cron: "0 0 * * *"
+    workflow_dispatch: ~
+
+permissions: read-all
+
+jobs:
+    test_ecosystem:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "lts/*"
+            - run: npm install
+            - id: tester
+              run: npm run test:ecosystem -- --plugin all
+            - if: steps.tester.outcome == 'failure' && github.head_ref == 'main'
+              env:
+                  DISCORD_WEBHOOK: ${{ secrets.DISCORD_CONTRIBUTORS_WEBHOOK }}
+              uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9 # v0.3.2
+              with:
+                  args: "Ecosystem tests failed."
@@ -0,0 +1,25 @@
+name: Update Test Ecosystem Plugins
+
+on:
+    schedule:
+        # “At 00:00 on Monday.” https://crontab.guru/#0_0_*_*_1
+        - cron: "0 0 * * 1"
+    workflow_dispatch: ~
+
+permissions: read-all
+
+jobs:
+    test_ecosystem:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "lts/*"
+            - run: npm install
+            - run: npm run test:ecosystem:update -- --plugin all
+            - uses: peter-evans/[email protected]
+              with:
+                  commit-message: "chore: update ecosystem plugins"
+                  title: "chore: update ecosystem plugins"
@@ -3,6 +3,7 @@
 test.js
 coverage/
 build/
+ecosystem/
 npm-debug.log
 yarn-error.log
 .pnpm-debug.log

@@ -25,6 +25,9 @@
         "tools/generate-formatter-examples.js",
       ],
       "ignoreDependencies": [
+        // Used in ecosystem.yml via nano-spawn shells
+        "@antfu/ni",
+        // Underlying code coverage engine used in unit tests
         "c8",
         // Optional peer dependency used for loading TypeScript configuration files
         "jiti",

@@ -70,6 +70,8 @@
     "test": "node Makefile.js test",
     "test:browser": "node Makefile.js cypress",
     "test:cli": "mocha",
+    "test:ecosystem": "node tools/test-ecosystem/index.mjs",
+    "test:ecosystem:update": "node tools/test-ecosystem/update.mjs",
     "test:fuzz": "node Makefile.js fuzz",
     "test:performance": "node Makefile.js perf",
     "test:emfile": "node tools/check-emfile-handling.js",
@@ -141,6 +143,7 @@
     "optionator": "^0.9.3"
   },
   "devDependencies": {
+    "@antfu/ni": "^24.3.0",
     "@arethetypeswrong/cli": "^0.18.0",
     "@babel/core": "^7.4.3",
     "@babel/preset-env": "^7.4.3",
@@ -188,6 +191,7 @@
     "metascraper-logo-favicon": "^5.25.7",
     "metascraper-title": "^5.25.7",
     "mocha": "^11.7.1",
+    "nano-spawn": "^1.0.1",
     "node-polyfill-webpack-plugin": "^1.0.3",
     "npm-license": "^0.3.3",
     "pirates": "^4.0.5",

@@ -0,0 +1,82 @@
+/**
+ * @fileoverview Data utilities for ecosystem tests and updates to data.
+ * @author Josh Goldberg
+ */
+
+//-----------------------------------------------------------------------------
+// Requirements
+//-----------------------------------------------------------------------------
+
+import chalk from "chalk";
+import util from "node:util";
+import path from "node:path";
+
+//-----------------------------------------------------------------------------
+// Types
+//-----------------------------------------------------------------------------
+
+/**
+ * Settings for how to clone, set up, and test an ecosystem plugin.
+ * @typedef {Object} PluginData
+ * @property {string} commit Hash to check out after cloning the plugin.
+ * @property {string} repository Repository URL to clone the plugin from.
+ */
+
+//-----------------------------------------------------------------------------
+// Constants
+//-----------------------------------------------------------------------------
+
+export const pluginDataFilePath = path.join(
+	import.meta.dirname,
+	"plugins-data.json",
+);
+
+//-----------------------------------------------------------------------------
+// Functions
+//-----------------------------------------------------------------------------
+
+/**
+ *
+ * @param {"test" | "update"} action
+ * @returns {[string, PluginData][]}
+ */
+export async function getPlugins(action) {
+	const { values } = util.parseArgs({
+		options: {
+			plugin: {
+				type: "string",
+				help: `The name of the plugin to ${action}, or 'all' for all plugins (default: 'all')`,
+			},
+		},
+	});
+
+	const { plugin: pluginRequested = "all" } = values;
+	const { default: pluginsData } = await import(pluginDataFilePath, {
+		with: { type: "json" },
+	});
+
+	if (pluginRequested !== "all" && !(pluginRequested in pluginsData)) {
+		console.error(`The plugin "${values.plugin}" is not supported.`);
+		console.error(
+			`Supported plugins are: ${["", ...Object.keys(pluginsData)].join(
+				"\n  ",
+			)}`,
+		);
+		console.error(
+			`Alternately, run without --plugin to ${action} all plugins.`,
+		);
+		process.exit(1);
+	}
+
+	const pluginsSelected =
+		pluginRequested === "all"
+			? Object.entries(pluginsData)
+			: [[pluginRequested, pluginsData[pluginRequested]]];
+
+	console.log(
+		`Plugins to ${action}:`,
+		chalk.bold(pluginsSelected.map(([key]) => key).join(", ")),
+	);
+
+	return { pluginsData, pluginsSelected };
+}
@@ -0,0 +1,146 @@
+/**
+ * @fileoverview A utility to test ecosystem plugin(s) against the built ESLint.
+ * @author Josh Goldberg
+ */
+
+//-----------------------------------------------------------------------------
+// Requirements
+//-----------------------------------------------------------------------------
+
+import chalk from "chalk";
+import spawn from "nano-spawn";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { getPlugins } from "./data.mjs";
+
+/**
+ * @typedef {import("./data").PluginSettings} PluginSettings
+ */
+
+//-----------------------------------------------------------------------------
+// Helpers
+//-----------------------------------------------------------------------------
+
+/**
+ * Runs ecosystem tests for a single plugin. It will:
+ * 1. Clone the plugin repository into a sandbox directory
+ * 2. Check out the plugin's commit to test on
+ * 3. Install the plugin's dependencies
+ * 4. Link the built ESLint into the plugin
+ * 5. Build, if the plugin defines a build script
+ * 6. Run tests
+ * This intentionally does not try/catch: any errors will be thrown.
+ *
+ * @param {string} pluginKey
+ * @param {PluginSettings} pluginSettings
+ */
+async function runTests(pluginKey, pluginSettings) {
+	const directory = path.join(
+		SANDBOX_DIRECTORY,
+		pluginKey
+			.replaceAll(/[^a-z-]/g, " ")
+			.trim()
+			.replaceAll(" ", "-"),
+	);
+	console.log(chalk.bold(`Testing ${pluginKey} in ${directory}`));
+
+	/**
+	 * Attempts to run a command in the plugin sandbox directory.
+	 * If it fails, any error stdout will be logged in red before a re-throw.
+	 * @param {string} command
+	 * @param {string[]} args
+	 */
+	const runCommand = async (command, ...args) => {
+		console.log(chalk.gray(`[${pluginKey}]`, [command, ...args].join(" ")));
+		try {
+			return await spawn(command, args, {
+				cwd: directory,
+			});
+		} catch (error) {
+			console.error(
+				chalk.red(`[${pluginKey}]`),
+				"stdout" in error ? error.stdout : error,
+			);
+			throw error;
+		}
+	};
+
+	// 1. Clone the plugin repository into a sandbox directory
+	await fs.mkdir(directory, { force: true });
+	await runCommand(
+		"git",
+		"clone",
+		pluginSettings.repository,
+		directory,
+		"--depth",
+		"1",
+	);
+
+	// 2. Check out the plugin's commit to test on
+	await runCommand("git", "fetch", "origin", pluginSettings.commit);
+	await runCommand("git", "checkout", pluginSettings.commit);
+
+	// 3. Install the plugin's dependencies
+	await runCommand("pwd");
+	await runCommand("ni");
+
+	// 4. Link the built ESLint into the plugin
+	await runCommand("npm", "link", "eslint");
+
+	const packageJsonPath = path.resolve(
+		process.cwd(),
+		path.join(directory, "package.json"),
+	);
+	const packageJson = await import(packageJsonPath, {
+		with: { type: "json" },
+	});
+
+	// 5. Build, if the plugin defines a build script
+	if (packageJson.default.scripts.build) {
+		await runCommand("nr", "build");
+	}
+
+	// 6. Run test
+	await runCommand("nr", "test");
+}
+
+//-----------------------------------------------------------------------------
+// Main
+//-----------------------------------------------------------------------------
+
+const { pluginsSelected } = await getPlugins("test");
+
+const SANDBOX_DIRECTORY = path.join(process.cwd(), "ecosystem");
+
+console.log(`Clearing existing sandbox directory: ${SANDBOX_DIRECTORY}`);
+await fs.rm(SANDBOX_DIRECTORY, {
+	force: true,
+	maxRetries: 8,
+	recursive: true,
+});
+await fs.mkdir(SANDBOX_DIRECTORY, { recursive: true });
+console.log("");
+
+const errors = [];
+
+// For each plugin to test, we try to runTests, recording thrown exceptions in errors
+for (const [pluginKey, pluginSettings] of pluginsSelected) {
+	try {
+		await runTests(pluginKey, pluginSettings);
+	} catch (error) {
+		errors.push({ error, pluginKey });
+	}
+
+	console.log("");
+}
+
+// If we had any errors, report them and exit as failed
+if (errors.length) {
+	console.error(chalk.red("Errors occurred while testing plugins:"));
+	for (const { error, pluginKey } of errors) {
+		console.error(`${chalk.bold.red(pluginKey)}: ${chalk.red(error)}`);
+	}
+	process.exitCode = 1;
+} else {
+	console.log(chalk.green("All tests completed successfully."));
+}
@@ -0,0 +1,26 @@
+{
+  "@eslint/css": {
+    "commit": "26b902c4f42cccb33d7f8119a3376773e0ad91bd",
+    "repository": "https://github.com/eslint/css"
+  },
+  "@eslint/json": {
+    "commit": "2d9722a9a3bab75301906d574266281305ec91df",
+    "repository": "https://github.com/eslint/json"
+  },
+  "@eslint/markdown": {
+    "commit": "614ef3e0bf539655170f202467119fbbbf8963f6",
+    "repository": "https://github.com/eslint/markdown"
+  },
+  "@eslint-community/eslint-plugin-eslint-comments": {
+    "commit": "d6869dfbb794437c00e0e27e6cc0dbdfd3c56fac",
+    "repository": "https://github.com/eslint-community/eslint-plugin-eslint-comments"
+  },
+  "eslint-plugin-unicorn": {
+    "commit": "f4eb2e731358d68b031d7fd42bf725a23f9d1e4d",
+    "repository": "https://github.com/sindresorhus/eslint-plugin-unicorn"
+  },
+  "eslint-plugin-vue": {
+    "commit": "fe0ce5a0590ee8b7aec57712f56f0d1d557035cb",
+    "repository": "https://github.com/vuejs/eslint-plugin-vue"
+  }
+}