[Phase 1] Track auto-upgrade metrics in Monorail

.changeset/auto-upgrade-metrics.md

@@ -0,0 +1,5 @@
+---
+'@shopify/cli-kit': patch
+---
+
+Track auto-upgrade events (trigger rate, success/failure, package manager, skip reason) in Monorail.

packages/cli-kit/src/public/node/hooks/postrun.test.ts

@@ -1,14 +1,21 @@
 import {autoUpgradeIfNeeded} from './postrun.js'
 import {versionToAutoUpgrade, runCLIUpgrade, getOutputUpdateCLIReminder} from '../upgrade.js'
 import {isMajorVersionChange} from '../version.js'
+import {inferPackageManagerForGlobalCLI} from '../is-global.js'
+import {addPublicMetadata} from '../metadata.js'
 import {mockAndCaptureOutput} from '../testing/output.js'
 import {describe, test, vi, expect, afterEach} from 'vitest'
 
 vi.mock('../upgrade.js')
 vi.mock('../version.js')
+vi.mock('../is-global.js')
+vi.mock('../metadata.js', () => ({
+  addPublicMetadata: vi.fn().mockResolvedValue(undefined),
+}))
 
 afterEach(() => {
   mockAndCaptureOutput().clear()
+  vi.mocked(addPublicMetadata).mockClear()
 })
 
 describe('autoUpgradeIfNeeded', () => {
@@ -35,6 +42,7 @@ describe('autoUpgradeIfNeeded', () => {
   test('calls runCLIUpgrade for minor/patch upgrade', async () => {
     vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0')
     vi.mocked(isMajorVersionChange).mockReturnValue(false)
+    vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm')
     vi.mocked(runCLIUpgrade).mockResolvedValue(undefined)
 
     await autoUpgradeIfNeeded()
@@ -46,11 +54,73 @@ describe('autoUpgradeIfNeeded', () => {
     const outputMock = mockAndCaptureOutput()
     vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0')
     vi.mocked(isMajorVersionChange).mockReturnValue(false)
+    vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm')
     vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed'))
-    vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`')
+    vi.mocked(getOutputUpdateCLIReminder).mockReturnValue(
+      '💡 Version 3.100.0 available! Run `npm install -g @shopify/cli@latest`',
+    )
 
     await autoUpgradeIfNeeded()
 
     expect(outputMock.warn()).toContain('3.100.0')
   })
+
+  test('records triggered metric for minor/patch upgrade', async () => {
+    vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0')
+    vi.mocked(isMajorVersionChange).mockReturnValue(false)
+    vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm')
+    vi.mocked(runCLIUpgrade).mockResolvedValue(undefined)
+
+    await autoUpgradeIfNeeded()
+
+    expect(addPublicMetadata).toHaveBeenCalledWith(expect.any(Function))
+    const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]())
+    expect(calls).toContainEqual(
+      expect.objectContaining({
+        cmd_all_auto_upgrade_triggered: true,
+        cmd_all_auto_upgrade_package_manager: 'npm',
+      }),
+    )
+  })
+
+  test('records triggered metric with major_version skipped_reason for major bump', async () => {
+    vi.mocked(versionToAutoUpgrade).mockReturnValue('4.0.0')
+    vi.mocked(isMajorVersionChange).mockReturnValue(true)
+    vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder')
+
+    await autoUpgradeIfNeeded()
+
+    const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]())
+    expect(calls).toContainEqual(
+      expect.objectContaining({
+        cmd_all_auto_upgrade_triggered: true,
+        cmd_all_auto_upgrade_skipped_reason: 'major_version',
+      }),
+    )
+  })
+
+  test('records success=true on successful upgrade', async () => {
+    vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0')
+    vi.mocked(isMajorVersionChange).mockReturnValue(false)
+    vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm')
+    vi.mocked(runCLIUpgrade).mockResolvedValue(undefined)
+
+    await autoUpgradeIfNeeded()
+
+    const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]())
+    expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: true}))
+  })
+
+  test('records success=false on failed upgrade', async () => {
+    vi.mocked(versionToAutoUpgrade).mockReturnValue('3.100.0')
+    vi.mocked(isMajorVersionChange).mockReturnValue(false)
+    vi.mocked(inferPackageManagerForGlobalCLI).mockReturnValue('npm')
+    vi.mocked(runCLIUpgrade).mockRejectedValue(new Error('upgrade failed'))
+    vi.mocked(getOutputUpdateCLIReminder).mockReturnValue('upgrade reminder')
+
+    await autoUpgradeIfNeeded()
+
+    const calls = vi.mocked(addPublicMetadata).mock.calls.map((call) => call[0]())
+    expect(calls).toContainEqual(expect.objectContaining({cmd_all_auto_upgrade_success: false}))
+  })
 })

packages/cli-kit/src/public/node/hooks/postrun.ts

@@ -4,6 +4,7 @@ import {outputDebug, outputWarn} from '../output.js'
 import {getOutputUpdateCLIReminder, runCLIUpgrade, versionToAutoUpgrade} from '../upgrade.js'
 import BaseCommand from '../base-command.js'
 import * as metadata from '../metadata.js'
+import {inferPackageManagerForGlobalCLI} from '../is-global.js'
 import {CLI_KIT_VERSION} from '../../common/version.js'
 import {isMajorVersionChange} from '../version.js'
 
@@ -40,19 +41,35 @@ export const hook: Hook.Postrun = async ({config, Command}) => {
  */
 export async function autoUpgradeIfNeeded(): Promise<void> {
   const newerVersion = versionToAutoUpgrade()
-  if (!newerVersion) return
+  if (!newerVersion) {
+    // versionToAutoUpgrade already logged the reason via outputDebug
+    return
+  }
+
   if (isMajorVersionChange(CLI_KIT_VERSION, newerVersion)) {
     outputWarn(getOutputUpdateCLIReminder(newerVersion))
+    await metadata.addPublicMetadata(() => ({
+      cmd_all_auto_upgrade_triggered: true,
+      cmd_all_auto_upgrade_skipped_reason: 'major_version',
+    }))
     return
   }
 
+  const packageManager = inferPackageManagerForGlobalCLI()
+  await metadata.addPublicMetadata(() => ({
+    cmd_all_auto_upgrade_triggered: true,
+    cmd_all_auto_upgrade_package_manager: packageManager,
+  }))
+
   try {
     await runCLIUpgrade()
+    await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: true}))
     // eslint-disable-next-line no-catch-all/no-catch-all
   } catch (error) {
     const errorMessage = `Auto-upgrade failed: ${error}`
     outputDebug(errorMessage)
     outputWarn(getOutputUpdateCLIReminder(newerVersion))
+    await metadata.addPublicMetadata(() => ({cmd_all_auto_upgrade_success: false}))
     // Report to Observe as a handled error without showing anything extra to the user
     const {sendErrorToBugsnag} = await import('../error-handler.js')
     await sendErrorToBugsnag(new Error(errorMessage), 'expected_error')

packages/cli-kit/src/public/node/is-global.test.ts

@@ -2,7 +2,7 @@ import {currentProcessIsGlobal, inferPackageManagerForGlobalCLI, installGlobalCL
 import {terminalSupportsPrompting} from './system.js'
 import {renderSelectPrompt} from './ui.js'
 import {globalCLIVersion} from './version.js'
-import {beforeEach, describe, expect, test, vi, afterEach} from 'vitest'
+import {describe, expect, test, vi, afterEach} from 'vitest'
 
 vi.mock('./system.js')
 vi.mock('./ui.js')

packages/cli-kit/src/public/node/monorail.ts

@@ -63,6 +63,12 @@ export interface Schemas {
       cmd_all_timing_prompts_ms?: Optional<number>
       cmd_all_timing_active_ms?: Optional<number>
 
+      // Auto-upgrade
+      cmd_all_auto_upgrade_triggered?: Optional<boolean>
+      cmd_all_auto_upgrade_skipped_reason?: Optional<string>
+      cmd_all_auto_upgrade_success?: Optional<boolean>
+      cmd_all_auto_upgrade_package_manager?: Optional<string>
+
       // Any extension related command
       cmd_extensions_binary_from_source?: Optional<boolean>
 

packages/cli/README.md

@@ -2765,16 +2765,16 @@ DESCRIPTION
 
 ## `shopify upgrade`
 
-Shows details on how to upgrade Shopify CLI.
+Upgrades Shopify CLI.
 
 ```
 USAGE
   $ shopify upgrade
 
 DESCRIPTION
-  Shows details on how to upgrade Shopify CLI.
+  Upgrades Shopify CLI.
 
-  Shows details on how to upgrade Shopify CLI.
+  Upgrades Shopify CLI using your package manager.
 ```
 
 ## `shopify version`

packages/cli/oclif.manifest.json

@@ -7822,8 +7822,8 @@
       ],
       "args": {
       },
-      "description": "Shows details on how to upgrade Shopify CLI.",
-      "descriptionWithMarkdown": "Shows details on how to upgrade Shopify CLI.",
+      "description": "Upgrades Shopify CLI using your package manager.",
+      "descriptionWithMarkdown": "Upgrades Shopify CLI using your package manager.",
       "enableJsonFlag": false,
       "flags": {
       },
@@ -7835,7 +7835,7 @@
       "pluginName": "@shopify/cli",
       "pluginType": "core",
       "strict": true,
-      "summary": "Shows details on how to upgrade Shopify CLI."
+      "summary": "Upgrades Shopify CLI."
     },
     "version": {
       "aliases": [

packages/cli/src/cli/commands/upgrade.test.ts

@@ -1,14 +1,13 @@
 import Upgrade from './upgrade.js'
+import {promptAutoUpgrade, runCLIUpgrade} from '@shopify/cli-kit/node/upgrade'
 import {describe, test, vi, expect} from 'vitest'
 
-vi.mock('@shopify/cli-kit/node/upgrade', () => ({
-  promptAutoUpgrade: vi.fn().mockResolvedValue(true),
-  runCLIUpgrade: vi.fn().mockResolvedValue(undefined),
-}))
+vi.mock('@shopify/cli-kit/node/upgrade')
 
 describe('upgrade command', () => {
   test('calls promptAutoUpgrade and runCLIUpgrade', async () => {
-    const {promptAutoUpgrade, runCLIUpgrade} = await import('@shopify/cli-kit/node/upgrade')
+    vi.mocked(promptAutoUpgrade).mockResolvedValue(true)
+    vi.mocked(runCLIUpgrade).mockResolvedValue(undefined)
 
     await Upgrade.run([], import.meta.url)