From d90246293088b00c4af1e31fd5bdcc1c527b73ff Mon Sep 17 00:00:00 2001 From: Ryan Bahan Date: Thu, 12 Mar 2026 12:08:37 -0600 Subject: [PATCH 1/3] Migrate cucumber tests to e2e and vitest Re-implement the three active cucumber test scenarios that were removed with the Cucumber infrastructure: - Command snapshot: Playwright e2e test comparing `shopify commands --tree` output against a checked-in snapshot file - GitHub Actions pinning: vitest repo health check ensuring third-party actions are pinned to SHA - Node deps sync: vitest repo health check ensuring shared dependencies use consistent versions across packages Co-Authored-By: Claude Opus 4.6 (1M context) --- package.json | 1 + packages/e2e/data/snapshots/commands.txt | 111 +++++++++++++++++++ packages/e2e/scripts/regenerate-snapshots.sh | 9 ++ packages/e2e/tests/commands.spec.ts | 37 +++++++ tests/repo-health.test.ts | 107 ++++++++++++++++++ vitest.workspace.json | 1 + 6 files changed, 266 insertions(+) create mode 100644 packages/e2e/data/snapshots/commands.txt create mode 100755 packages/e2e/scripts/regenerate-snapshots.sh create mode 100644 packages/e2e/tests/commands.spec.ts create mode 100644 tests/repo-health.test.ts diff --git a/package.json b/package.json index 880b55179ea..1a33e7d5a83 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "shopify:run": "node packages/cli/bin/dev.js", "shopify": "nx build cli && node packages/cli/bin/dev.js", "test:e2e": "nx run-many --target=build --projects=cli,create-app --skip-nx-cache && pnpm --filter e2e exec playwright test", + "test:regenerate-snapshots": "packages/e2e/scripts/regenerate-snapshots.sh", "test:unit": "pnpm vitest run", "test": "pnpm vitest run", "type-check:affected": "nx affected --target=type-check", diff --git a/packages/e2e/data/snapshots/commands.txt b/packages/e2e/data/snapshots/commands.txt new file mode 100644 index 00000000000..bfd6e031fcb --- /dev/null +++ b/packages/e2e/data/snapshots/commands.txt @@ -0,0 +1,111 @@ +├─ app +│ ├─ build +│ ├─ bulk +│ │ ├─ cancel +│ │ ├─ execute +│ │ └─ status +│ ├─ config +│ │ ├─ link +│ │ ├─ pull +│ │ └─ use +│ ├─ deploy +│ ├─ dev +│ │ └─ clean +│ ├─ env +│ │ ├─ pull +│ │ └─ show +│ ├─ execute +│ ├─ function +│ │ ├─ build +│ │ ├─ info +│ │ ├─ replay +│ │ ├─ run +│ │ ├─ schema +│ │ └─ typegen +│ ├─ generate +│ │ └─ extension +│ ├─ import-custom-data-definitions +│ ├─ import-extensions +│ ├─ info +│ ├─ init +│ ├─ logs +│ │ └─ sources +│ ├─ release +│ ├─ versions +│ │ └─ list +│ └─ webhook +│ └─ trigger +├─ auth +│ ├─ login +│ └─ logout +├─ commands +├─ config +│ └─ autocorrect +│ ├─ off +│ ├─ on +│ └─ status +├─ help +├─ hydrogen +│ ├─ build +│ ├─ check +│ ├─ codegen +│ ├─ customer-account-push +│ ├─ debug +│ │ └─ cpu +│ ├─ deploy +│ ├─ dev +│ ├─ env +│ │ ├─ list +│ │ ├─ pull +│ │ └─ push +│ ├─ generate +│ │ ├─ route +│ │ └─ routes +│ ├─ init +│ ├─ link +│ ├─ list +│ ├─ login +│ ├─ logout +│ ├─ preview +│ ├─ setup +│ │ ├─ css +│ │ ├─ markets +│ │ └─ vite +│ ├─ shortcut +│ ├─ unlink +│ └─ upgrade +├─ organization +│ └─ list +├─ plugins +│ ├─ add +│ ├─ inspect +│ ├─ install +│ ├─ link +│ ├─ remove +│ ├─ reset +│ ├─ uninstall +│ ├─ unlink +│ └─ update +├─ search +├─ theme +│ ├─ check +│ ├─ console +│ ├─ delete +│ ├─ dev +│ ├─ duplicate +│ ├─ info +│ ├─ init +│ ├─ language-server +│ ├─ list +│ ├─ metafields +│ │ └─ pull +│ ├─ open +│ ├─ package +│ ├─ profile +│ ├─ publish +│ ├─ pull +│ ├─ push +│ ├─ rename +│ └─ share +├─ upgrade +└─ version diff --git a/packages/e2e/scripts/regenerate-snapshots.sh b/packages/e2e/scripts/regenerate-snapshots.sh new file mode 100755 index 00000000000..70db24309d2 --- /dev/null +++ b/packages/e2e/scripts/regenerate-snapshots.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# enter this dir so that we can run this script from the top-level +cd "$(dirname "$0")" + +# regenerate commands snapshot file +node ../../cli/bin/dev.js commands --tree > ../data/snapshots/commands.txt diff --git a/packages/e2e/tests/commands.spec.ts b/packages/e2e/tests/commands.spec.ts new file mode 100644 index 00000000000..1a9612bc464 --- /dev/null +++ b/packages/e2e/tests/commands.spec.ts @@ -0,0 +1,37 @@ +import {cliFixture as test} from '../setup/cli.js' +import {expect} from '@playwright/test' +import * as fs from 'fs/promises' +import * as path from 'path' +import {fileURLToPath} from 'url' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const snapshotPath = path.join(__dirname, '../data/snapshots/commands.txt') + +const errorMessage = ` +SNAPSHOT TEST FAILED! + +The result of 'shopify commands --tree' has changed! We run this to check that +all commands can load successfully. + +It's normal to see this test fail when you add or remove a command in the CLI. +In this case you can run this command to regenerate the snapshot file: + +$ pnpm test:regenerate-snapshots + +Then you can commit this change and this test will pass. + +If instead you didn't mean to change a command, UH OH. Check the commands in +the diff below and figure out what is broken. +` + +const normalize = (value: string) => value.replace(/\r\n/g, '\n').trimEnd() + +test.describe('Command snapshot', () => { + test('shopify commands --tree matches snapshot', async ({cli}) => { + const result = await cli.exec(['commands', '--tree']) + expect(result.exitCode).toBe(0) + + const snapshot = await fs.readFile(snapshotPath, {encoding: 'utf8'}) + expect(normalize(result.stdout), errorMessage).toBe(normalize(snapshot)) + }) +}) diff --git a/tests/repo-health.test.ts b/tests/repo-health.test.ts new file mode 100644 index 00000000000..3c5aa92ef83 --- /dev/null +++ b/tests/repo-health.test.ts @@ -0,0 +1,107 @@ +import {describe, test, expect} from 'vitest' +import * as fs from 'fs/promises' +import * as path from 'path' +import glob from 'fast-glob' + +const repoRoot = path.join(__dirname, '..') + +describe('GitHub Actions pinning', () => { + test('all non-official actions are pinned to SHA', async () => { + const workflowDir = path.join(repoRoot, '.github/workflows') + const workflowFiles = await glob('*.yml', {cwd: workflowDir, absolute: true}) + expect(workflowFiles.length).toBeGreaterThan(0) + + const allActions: string[] = [] + for (const file of workflowFiles) { + const content = await fs.readFile(file, 'utf-8') + const matches = content.match(/uses:\s+\S+/g) ?? [] + allActions.push(...matches.map((m) => m.split(/\s+/)[1]!)) + } + + const thirdParty = allActions.filter( + (action) => !action.startsWith('actions/') && !action.startsWith('./') && !action.startsWith('Shopify/'), + ) + + const unpinned = thirdParty.filter((action) => !action.match(/^[^@]+@[0-9a-f]+/)) + + expect(unpinned, [ + 'The following unofficial GitHub actions have not been pinned:\n', + ...unpinned.map((el) => ` - ${el}\n`), + '\nRun bin/pin-github-actions.js, verify the action is not doing anything malicious, then commit your changes.', + ].join('')).toHaveLength(0) + }) +}) + +describe('Node dependency version sync', () => { + const sharedDependencies = [ + '@babel/core', + '@oclif/core', + '@shopify/cli-kit', + '@types/node', + '@typescript-eslint/parser', + 'esbuild', + 'execa', + 'fast-glob', + 'graphql', + 'graphql-request', + 'graphql-tag', + 'ink', + 'liquidjs', + 'node-fetch', + 'typescript', + 'vite', + 'vitest', + 'zod', + ] + + interface PackageJson { + dependencies?: Record + devDependencies?: Record + peerDependencies?: Record + resolutions?: Record + } + + test('shared dependencies are on the same version across packages', async () => { + const packageJsonPaths = await glob('packages/*/package.json', {cwd: repoRoot, absolute: true}) + const packageJsonMap: Record = {} + + for (const pkgPath of packageJsonPaths) { + const name = path.dirname(pkgPath).split('/').pop()! + packageJsonMap[name] = JSON.parse(await fs.readFile(pkgPath, 'utf-8')) as PackageJson + } + packageJsonMap['root'] = JSON.parse(await fs.readFile(path.join(repoRoot, 'package.json'), 'utf-8')) as PackageJson + + const different: {dep: string; versions: {packageName: string; version: string}[]}[] = [] + + for (const dep of sharedDependencies) { + const depVersions: {packageName: string; version: string}[] = [] + + for (const [packageName, json] of Object.entries(packageJsonMap)) { + const version = + json.dependencies?.[dep] ?? + json.devDependencies?.[dep] ?? + json.peerDependencies?.[dep] ?? + json.resolutions?.[dep] + if (version) { + depVersions.push({packageName, version: version.replace(/^\^/, '')}) + } + } + + const uniqueVersions = [...new Set(depVersions.map((v) => v.version))] + if (uniqueVersions.length > 1) { + different.push({dep, versions: depVersions}) + } + } + + const errorMessage = [ + 'The following node dependencies are on different versions across packages:\n\n', + ...different.map( + ({dep, versions}) => + ` - ${dep}:\n${versions.map(({packageName, version}) => ` - ${packageName}: ${version}`).join('\n')}`, + ), + '\n\nPlease make sure they are all on the same version.', + ].join('') + + expect(different, errorMessage).toHaveLength(0) + }) +}) diff --git a/vitest.workspace.json b/vitest.workspace.json index a34f6c8699d..bb2c57e8561 100644 --- a/vitest.workspace.json +++ b/vitest.workspace.json @@ -1,4 +1,5 @@ [ + ".", "packages/app", "packages/cli", "packages/cli-kit", From a5efad0ce882a61ef2d5052b5ec7c1a275bde166 Mon Sep 17 00:00:00 2001 From: Ryan Bahan Date: Thu, 12 Mar 2026 12:23:42 -0600 Subject: [PATCH 2/3] Fix lint and type-check for migrated tests - Add eslint-disable for no-restricted-imports and no-await-in-loop (repo health checks intentionally use Node stdlib directly) - Fix id-length violations (m -> match, v -> ver) - Add tests/tsconfig.json so eslint projectService can parse root tests - Add eslint-disable no-restricted-imports to e2e command snapshot test Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/e2e/tests/commands.spec.ts | 1 + tests/repo-health.test.ts | 22 +++++++++++++--------- tests/tsconfig.json | 11 +++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 tests/tsconfig.json diff --git a/packages/e2e/tests/commands.spec.ts b/packages/e2e/tests/commands.spec.ts index 1a9612bc464..d650bda879a 100644 --- a/packages/e2e/tests/commands.spec.ts +++ b/packages/e2e/tests/commands.spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-restricted-imports */ import {cliFixture as test} from '../setup/cli.js' import {expect} from '@playwright/test' import * as fs from 'fs/promises' diff --git a/tests/repo-health.test.ts b/tests/repo-health.test.ts index 3c5aa92ef83..b83e93cb03b 100644 --- a/tests/repo-health.test.ts +++ b/tests/repo-health.test.ts @@ -1,7 +1,8 @@ +/* eslint-disable no-restricted-imports, no-await-in-loop */ import {describe, test, expect} from 'vitest' +import glob from 'fast-glob' import * as fs from 'fs/promises' import * as path from 'path' -import glob from 'fast-glob' const repoRoot = path.join(__dirname, '..') @@ -15,7 +16,7 @@ describe('GitHub Actions pinning', () => { for (const file of workflowFiles) { const content = await fs.readFile(file, 'utf-8') const matches = content.match(/uses:\s+\S+/g) ?? [] - allActions.push(...matches.map((m) => m.split(/\s+/)[1]!)) + allActions.push(...matches.map((match) => match.split(/\s+/)[1]!)) } const thirdParty = allActions.filter( @@ -24,11 +25,14 @@ describe('GitHub Actions pinning', () => { const unpinned = thirdParty.filter((action) => !action.match(/^[^@]+@[0-9a-f]+/)) - expect(unpinned, [ - 'The following unofficial GitHub actions have not been pinned:\n', - ...unpinned.map((el) => ` - ${el}\n`), - '\nRun bin/pin-github-actions.js, verify the action is not doing anything malicious, then commit your changes.', - ].join('')).toHaveLength(0) + expect( + unpinned, + [ + 'The following unofficial GitHub actions have not been pinned:\n', + ...unpinned.map((el) => ` - ${el}\n`), + '\nRun bin/pin-github-actions.js, verify the action is not doing anything malicious, then commit your changes.', + ].join(''), + ).toHaveLength(0) }) }) @@ -69,7 +73,7 @@ describe('Node dependency version sync', () => { const name = path.dirname(pkgPath).split('/').pop()! packageJsonMap[name] = JSON.parse(await fs.readFile(pkgPath, 'utf-8')) as PackageJson } - packageJsonMap['root'] = JSON.parse(await fs.readFile(path.join(repoRoot, 'package.json'), 'utf-8')) as PackageJson + packageJsonMap.root = JSON.parse(await fs.readFile(path.join(repoRoot, 'package.json'), 'utf-8')) as PackageJson const different: {dep: string; versions: {packageName: string; version: string}[]}[] = [] @@ -87,7 +91,7 @@ describe('Node dependency version sync', () => { } } - const uniqueVersions = [...new Set(depVersions.map((v) => v.version))] + const uniqueVersions = [...new Set(depVersions.map((ver) => ver.version))] if (uniqueVersions.length > 1) { different.push({dep, versions: depVersions}) } diff --git a/tests/tsconfig.json b/tests/tsconfig.json new file mode 100644 index 00000000000..ec6061cf71d --- /dev/null +++ b/tests/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../configurations/tsconfig.json", + "compilerOptions": { + "composite": false, + "declaration": false, + "sourceMap": false, + "inlineSources": false, + "noEmit": true + }, + "include": ["./**/*.ts"] +} From 374974f9f05aa4819db69ad645645e2875a4794e Mon Sep 17 00:00:00 2001 From: Ryan Bahan Date: Thu, 12 Mar 2026 12:28:41 -0600 Subject: [PATCH 3/3] Move repo health tests into packages/cli, fix knip - Move tests/repo-health.test.ts to packages/cli/src/cli/ so it lives in an existing vitest workspace project (no root tsconfig needed) - Remove tests/tsconfig.json and root "." vitest workspace entry - Add packages/e2e/scripts/* to knip ignoreBinaries to fix CI failure Co-Authored-By: Claude Opus 4.6 (1M context) --- package.json | 1 + {tests => packages/cli/src/cli}/repo-health.test.ts | 2 +- tests/tsconfig.json | 11 ----------- vitest.workspace.json | 1 - 4 files changed, 2 insertions(+), 13 deletions(-) rename {tests => packages/cli/src/cli}/repo-health.test.ts (98%) delete mode 100644 tests/tsconfig.json diff --git a/package.json b/package.json index 1a33e7d5a83..f23b3d394eb 100644 --- a/package.json +++ b/package.json @@ -173,6 +173,7 @@ ], "ignoreBinaries": [ "bin/*", + "packages/e2e/scripts/*", "shopify", "shopify-nightly", "brew", diff --git a/tests/repo-health.test.ts b/packages/cli/src/cli/repo-health.test.ts similarity index 98% rename from tests/repo-health.test.ts rename to packages/cli/src/cli/repo-health.test.ts index b83e93cb03b..7bfe2651db3 100644 --- a/tests/repo-health.test.ts +++ b/packages/cli/src/cli/repo-health.test.ts @@ -4,7 +4,7 @@ import glob from 'fast-glob' import * as fs from 'fs/promises' import * as path from 'path' -const repoRoot = path.join(__dirname, '..') +const repoRoot = path.join(__dirname, '../../../..') describe('GitHub Actions pinning', () => { test('all non-official actions are pinned to SHA', async () => { diff --git a/tests/tsconfig.json b/tests/tsconfig.json deleted file mode 100644 index ec6061cf71d..00000000000 --- a/tests/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../configurations/tsconfig.json", - "compilerOptions": { - "composite": false, - "declaration": false, - "sourceMap": false, - "inlineSources": false, - "noEmit": true - }, - "include": ["./**/*.ts"] -} diff --git a/vitest.workspace.json b/vitest.workspace.json index bb2c57e8561..a34f6c8699d 100644 --- a/vitest.workspace.json +++ b/vitest.workspace.json @@ -1,5 +1,4 @@ [ - ".", "packages/app", "packages/cli", "packages/cli-kit",