diff --git a/.changeset/auto-upgrade-metrics.md b/.changeset/auto-upgrade-metrics.md new file mode 100644 index 0000000000..0bbb69d6c2 --- /dev/null +++ b/.changeset/auto-upgrade-metrics.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': patch +--- + +Track auto-upgrade events (trigger rate, success/failure, package manager, skip reason) in Monorail. diff --git a/packages/cli-kit/src/public/node/hooks/postrun.test.ts b/packages/cli-kit/src/public/node/hooks/postrun.test.ts index 4d2bc6ff6d..a306d80191 100644 --- a/packages/cli-kit/src/public/node/hooks/postrun.test.ts +++ b/packages/cli-kit/src/public/node/hooks/postrun.test.ts @@ -1,14 +1,21 @@ import {autoUpgradeIfNeeded} from './postrun.js' import {versionToAutoUpgrade, runCLIUpgrade, getOutputUpdateCLIReminder} from '../upgrade.js' import {isMajorVersionChange} from '../version.js' +import {inferPackageManagerForGlobalCLI} from '../is-global.js' +import {addPublicMetadata} from '../metadata.js' import {mockAndCaptureOutput} from '../testing/output.js' import {describe, test, vi, expect, afterEach} from 'vitest' vi.mock('../upgrade.js') vi.mock('../version.js') +vi.mock('../is-global.js') +vi.mock('../metadata.js', () => ({ + addPublicMetadata: vi.fn().mockResolvedValue(undefined), +})) afterEach(() => { mockAndCaptureOutput().clear() + vi.mocked(addPublicMetadata).mockClear() }) describe('autoUpgradeIfNeeded', () => { @@ -35,6 +42,7 @@ describe('autoUpgradeIfNeeded', () => { test('calls runCLIUpgrade for minor/patch upgrade', async () => { vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) await autoUpgradeIfNeeded() @@ -46,11 +54,73 @@ describe('autoUpgradeIfNeeded', () => { const outputMock = mockAndCaptureOutput() vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed')) - vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`') + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue( + '💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`', + ) await autoUpgradeIfNeeded() expect(outputMock.warn()).toContain('3.100.0') }) + + test('records triggered metric for minor/patch upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) + + await autoUpgradeIfNeeded() + + expect(addPublicMetadata).toHaveBeenCalledWith(expect.any(Function)) + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual( + expect.objectContaining({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_package_manager: 'npm', + }), + ) + }) + + test('records triggered metric with major_version skipped_reason for major bump', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('4.0.0') + vi.mocked(isMajorVersionChange).mockReturnValue(true) + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder') + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual( + expect.objectContaining({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_skipped_reason: 'major_version', + }), + ) + }) + + test('records success=true on successful upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: true})) + }) + + test('records success=false on failed upgrade', async () => { + vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0') + vi.mocked(isMajorVersionChange).mockReturnValue(false) + vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm') + vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed')) + vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder') + + await autoUpgradeIfNeeded() + + const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]()) + expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: false})) + }) }) diff --git a/packages/cli-kit/src/public/node/hooks/postrun.ts b/packages/cli-kit/src/public/node/hooks/postrun.ts index a768500793..38d7311ed2 100644 --- a/packages/cli-kit/src/public/node/hooks/postrun.ts +++ b/packages/cli-kit/src/public/node/hooks/postrun.ts @@ -4,6 +4,7 @@ import {outputDebug, outputWarn} from '../output.js' import {getOutputUpdateCLIReminder, runCLIUpgrade, versionToAutoUpgrade} from '../upgrade.js' import BaseCommand from '../base-command.js' import * as metadata from '../metadata.js' +import {inferPackageManagerForGlobalCLI} from '../is-global.js' import {CLI_KIT_VERSION} from '../../common/version.js' import {isMajorVersionChange} from '../version.js' @@ -40,19 +41,35 @@ export const hook: Hook.Postrun = async ({config, Command}) => { */ export async function autoUpgradeIfNeeded(): Promise { const newerVersion = versionToAutoUpgrade() - if (!newerVersion) return + if (!newerVersion) { + // versionToAutoUpgrade already logged the reason via outputDebug + return + } + if (isMajorVersionChange(CLI_KIT_VERSION, newerVersion)) { outputWarn(getOutputUpdateCLIReminder(newerVersion)) + await metadata.addPublicMetadata(() => ({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_skipped_reason: 'major_version', + })) return } + const packageManager = inferPackageManagerForGlobalCLI() + await metadata.addPublicMetadata(() => ({ + cmd_all_auto_upgrade_triggered: true, + cmd_all_auto_upgrade_package_manager: packageManager, + })) + try { await runCLIUpgrade() + await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: true})) // eslint-disable-next-line no-catch-all/no-catch-all } catch (error) { const errorMessage = `Auto-upgrade failed: ${error}` outputDebug(errorMessage) outputWarn(getOutputUpdateCLIReminder(newerVersion)) + await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: false})) // Report to Observe as a handled error without showing anything extra to the user const {sendErrorToBugsnag} = await import('../error-handler.js') await sendErrorToBugsnag(new Error(errorMessage), 'expected_error') diff --git a/packages/cli-kit/src/public/node/is-global.test.ts b/packages/cli-kit/src/public/node/is-global.test.ts index 5142ba9d77..5ed18d5186 100644 --- a/packages/cli-kit/src/public/node/is-global.test.ts +++ b/packages/cli-kit/src/public/node/is-global.test.ts @@ -2,7 +2,7 @@ import {currentProcessIsGlobal, inferPackageManagerForGlobalCLI, installGlobalCL import {terminalSupportsPrompting} from './system.js' import {renderSelectPrompt} from './ui.js' import {globalCLIVersion} from './version.js' -import {beforeEach, describe, expect, test, vi, afterEach} from 'vitest' +import {describe, expect, test, vi, afterEach} from 'vitest' vi.mock('./system.js') vi.mock('./ui.js') diff --git a/packages/cli-kit/src/public/node/monorail.ts b/packages/cli-kit/src/public/node/monorail.ts index b687ec9fee..a14be863b2 100644 --- a/packages/cli-kit/src/public/node/monorail.ts +++ b/packages/cli-kit/src/public/node/monorail.ts @@ -63,6 +63,12 @@ export interface Schemas { cmd_all_timing_prompts_ms?: Optional cmd_all_timing_active_ms?: Optional + // Auto-upgrade + cmd_all_auto_upgrade_triggered?: Optional + cmd_all_auto_upgrade_skipped_reason?: Optional + cmd_all_auto_upgrade_success?: Optional + cmd_all_auto_upgrade_package_manager?: Optional + // Any extension related command cmd_extensions_binary_from_source?: Optional diff --git a/packages/cli/README.md b/packages/cli/README.md index 0ef528b2e4..07fb8dee74 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -2765,16 +2765,16 @@ DESCRIPTION ## `shopify upgrade` -Shows details on how to upgrade Shopify CLI. +Upgrades Shopify CLI. ``` USAGE $ shopify upgrade DESCRIPTION - Shows details on how to upgrade Shopify CLI. + Upgrades Shopify CLI. - Shows details on how to upgrade Shopify CLI. + Upgrades Shopify CLI using your package manager. ``` ## `shopify version` diff --git a/packages/cli/oclif.manifest.json b/packages/cli/oclif.manifest.json index 8a053bd1ea..78675c99fc 100644 --- a/packages/cli/oclif.manifest.json +++ b/packages/cli/oclif.manifest.json @@ -7822,8 +7822,8 @@ ], "args": { }, - "description": "Shows details on how to upgrade Shopify CLI.", - "descriptionWithMarkdown": "Shows details on how to upgrade Shopify CLI.", + "description": "Upgrades Shopify CLI using your package manager.", + "descriptionWithMarkdown": "Upgrades Shopify CLI using your package manager.", "enableJsonFlag": false, "flags": { }, @@ -7835,7 +7835,7 @@ "pluginName": "@shopify/cli", "pluginType": "core", "strict": true, - "summary": "Shows details on how to upgrade Shopify CLI." + "summary": "Upgrades Shopify CLI." }, "version": { "aliases": [ diff --git a/packages/cli/src/cli/commands/upgrade.test.ts b/packages/cli/src/cli/commands/upgrade.test.ts index 61e04ba205..947195bb63 100644 --- a/packages/cli/src/cli/commands/upgrade.test.ts +++ b/packages/cli/src/cli/commands/upgrade.test.ts @@ -1,14 +1,13 @@ import Upgrade from './upgrade.js' +import {promptAutoUpgrade, runCLIUpgrade} from '@shopify/cli-kit/node/upgrade' import {describe, test, vi, expect} from 'vitest' -vi.mock('@shopify/cli-kit/node/upgrade', () => ({ - promptAutoUpgrade: vi.fn().mockResolvedValue(true), - runCLIUpgrade: vi.fn().mockResolvedValue(undefined), -})) +vi.mock('@shopify/cli-kit/node/upgrade') describe('upgrade command', () => { test('calls promptAutoUpgrade and runCLIUpgrade', async () => { - const {promptAutoUpgrade, runCLIUpgrade} = await import('@shopify/cli-kit/node/upgrade') + vi.mocked(promptAutoUpgrade).mockResolvedValue(true) + vi.mocked(runCLIUpgrade).mockResolvedValue(undefined) await Upgrade.run([], import.meta.url)