perf(providers): memoize SDK clients where the pool is per-client (bedrock, vllm)

Generalize the Anthropic client cache into one shared memoizer
(providers/client-cache.ts) and apply it only where each new client owns its own
connection pool — so reuse actually keeps connections warm:

- bedrock: AWS SDK clients hold a per-client connection pool (reuse is the AWS
  best practice). Keyed by region + credential identity.
- vllm: a pinned endpoint creates its own undici Agent per call; key by the
  resolved IP so DNS re-validation still runs each request.
- anthropic + azure-anthropic: migrated onto the shared memoizer.

Deliberately NOT applied to the OpenAI-compatible providers, groq, cerebras, or
google: their SDKs share a process-global keep-alive pool (Node openai-sdk module
singleton agent; anthropic/global undici), so a fresh client per request already
reuses connections and memoization would add complexity with ~no benefit. litellm
uses a plain shared-agent client (no pinning) and is likewise skipped.

Bounded LRU (max 1000, 30m idle TTL) with no close-on-eviction, avoiding the
unbounded-growth and eviction-closes-in-use-client failure modes seen in similar
client caches.

Author: waleedlatif1

apps/sim/providers/anthropic/client-cache.test.ts

@@ -1,8 +1,8 @@
 import Anthropic from '@anthropic-ai/sdk'
 import { createLogger } from '@sim/logger'
 import type { StreamingExecution } from '@/executor/types'
-import { getCachedAnthropicClient } from '@/providers/anthropic/client-cache'
 import { executeAnthropicProviderRequest } from '@/providers/anthropic/core'
+import { getCachedProviderClient } from '@/providers/client-cache'
 import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
 import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types'
 
@@ -23,8 +23,8 @@ export const anthropicProvider: ProviderConfig = {
       providerId: 'anthropic',
       providerLabel: 'Anthropic',
       createClient: (apiKey, useNativeStructuredOutputs) => {
-        const cacheKey = `${apiKey}::${useNativeStructuredOutputs ? 'beta' : 'default'}`
-        return getCachedAnthropicClient(
+        const cacheKey = `anthropic::${apiKey}::${useNativeStructuredOutputs ? 'beta' : 'default'}`
+        return getCachedProviderClient(
           cacheKey,
           () =>
             new Anthropic({

apps/sim/providers/anthropic/client-cache.ts

@@ -3,8 +3,8 @@ import { createLogger } from '@sim/logger'
 import { env } from '@/lib/core/config/env'
 import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server'
 import type { StreamingExecution } from '@/executor/types'
-import { getCachedAnthropicClient } from '@/providers/anthropic/client-cache'
 import { executeAnthropicProviderRequest } from '@/providers/anthropic/core'
+import { getCachedProviderClient } from '@/providers/client-cache'
 import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
 import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types'
 
@@ -73,13 +73,14 @@ export const azureAnthropicProvider: ProviderConfig = {
         providerLabel: 'Azure Anthropic',
         createClient: (apiKey, useNativeStructuredOutputs) => {
           const cacheKey = [
+            'azure-anthropic',
             apiKey,
             baseURL,
             anthropicVersion,
             pinnedIP ?? 'no-pin',
             useNativeStructuredOutputs ? 'beta' : 'default',
           ].join('::')
-          return getCachedAnthropicClient(
+          return getCachedProviderClient(
             cacheKey,
             () =>
               new Anthropic({

apps/sim/providers/anthropic/index.ts

@@ -24,6 +24,7 @@ import {
   generateToolUseId,
   getBedrockInferenceProfileId,
 } from '@/providers/bedrock/utils'
+import { getCachedProviderClient } from '@/providers/client-cache'
 import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
 import { createStreamingExecution } from '@/providers/streaming-execution'
 import { enrichLastModelSegment } from '@/providers/trace-enrichment'
@@ -138,7 +139,14 @@ export const bedrockProvider: ProviderConfig = {
       }
     }
 
-    const client = new BedrockRuntimeClient(clientConfig)
+    // Memoized: each BedrockRuntimeClient owns its own connection pool (AWS SDK
+    // best practice is to reuse the client), so reusing it keeps connections warm
+    // across requests. Keyed by region + credential identity (a rotated key pair
+    // changes the access key id and so yields a fresh client).
+    const client = getCachedProviderClient(
+      `bedrock::${region}::${request.bedrockAccessKeyId ?? 'default-chain'}`,
+      () => new BedrockRuntimeClient(clientConfig)
+    )
 
     const messages: BedrockMessage[] = []
     const systemContent: SystemContentBlock[] = []

apps/sim/providers/azure-anthropic/index.ts

@@ -0,0 +1,107 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+import { getCachedProviderClient } from '@/providers/client-cache'
+
+/**
+ * Builds a fresh fake "client" object on every call so identity comparisons
+ * (`toBe`) tell us whether the cache returned the memoized instance or a new one
+ * from the factory. We never construct a real SDK client — these tests exercise
+ * the cache, not any provider SDK.
+ */
+function makeFactory() {
+  return vi.fn(() => ({}) as object)
+}
+
+/**
+ * Generates a unique suffix per test so distinct tests never collide on cache
+ * keys. The cache util exposes no reset hook, so isolation is achieved by
+ * namespacing keys rather than clearing shared state.
+ */
+let keyCounter = 0
+function uniqueNs(): string {
+  keyCounter += 1
+  return `ns-${keyCounter}-${Date.now()}`
+}
+
+describe('getCachedProviderClient', () => {
+  it('returns the SAME instance for an identical key and runs the factory once (memoized)', () => {
+    const key = `anthropic::${uniqueNs()}::default`
+    const factory = makeFactory()
+
+    const first = getCachedProviderClient(key, factory)
+    const second = getCachedProviderClient(key, factory)
+
+    expect(second).toBe(first)
+    expect(factory).toHaveBeenCalledTimes(1)
+  })
+
+  it('returns a DIFFERENT instance for a different apiKey (tenant isolation)', () => {
+    const ns = uniqueNs()
+    const factoryA = makeFactory()
+    const factoryB = makeFactory()
+
+    const tenantA = getCachedProviderClient(`anthropic::${ns}-tenant-a::default`, factoryA)
+    const tenantB = getCachedProviderClient(`anthropic::${ns}-tenant-b::default`, factoryB)
+
+    expect(tenantB).not.toBe(tenantA)
+    expect(factoryA).toHaveBeenCalledTimes(1)
+    expect(factoryB).toHaveBeenCalledTimes(1)
+  })
+
+  it('namespaces by provider: the same apiKey under different provider prefixes does not collide', () => {
+    const ns = uniqueNs()
+    const apiKey = `${ns}-shared-key`
+    const anthropicFactory = makeFactory()
+    const bedrockFactory = makeFactory()
+
+    const anthropicClient = getCachedProviderClient(`anthropic::${apiKey}`, anthropicFactory)
+    const bedrockClient = getCachedProviderClient(`bedrock::${apiKey}`, bedrockFactory)
+
+    expect(bedrockClient).not.toBe(anthropicClient)
+  })
+
+  it('treats every distinct key dimension as a distinct client', () => {
+    const ns = uniqueNs()
+    const base = `azure-anthropic::${ns}-key::https://a.example.com::2023-06-01::10.0.0.1::default`
+    const baseFactory = makeFactory()
+    const baseClient = getCachedProviderClient(base, baseFactory)
+
+    const variants = [
+      `azure-anthropic::${ns}-key::https://b.example.com::2023-06-01::10.0.0.1::default`,
+      `azure-anthropic::${ns}-key::https://a.example.com::2024-10-22::10.0.0.1::default`,
+      `azure-anthropic::${ns}-key::https://a.example.com::2023-06-01::10.0.0.2::default`,
+      `azure-anthropic::${ns}-key::https://a.example.com::2023-06-01::no-pin::default`,
+      `azure-anthropic::${ns}-key::https://a.example.com::2023-06-01::10.0.0.1::beta`,
+    ]
+
+    for (const key of variants) {
+      const factory = makeFactory()
+      const client = getCachedProviderClient(key, factory)
+      expect(client).not.toBe(baseClient)
+      expect(factory).toHaveBeenCalledTimes(1)
+    }
+  })
+
+  it('evicts the least-recently-used entry once the cache cap is exceeded', () => {
+    const ns = uniqueNs()
+    const CAP = 1_000
+
+    const oldestKey = `evict::${ns}::0`
+    const oldestFactory = makeFactory()
+    getCachedProviderClient(oldestKey, oldestFactory)
+    expect(oldestFactory).toHaveBeenCalledTimes(1)
+
+    // Fill the remaining capacity, then push one past the cap. The oldest key has
+    // not been touched since insertion, so it is the LRU eviction victim.
+    for (let i = 1; i <= CAP; i += 1) {
+      getCachedProviderClient(`evict::${ns}::${i}`, makeFactory())
+    }
+
+    const reFactory = makeFactory()
+    getCachedProviderClient(oldestKey, reFactory)
+    expect(reFactory).toHaveBeenCalledTimes(1)
+    expect(oldestFactory).toHaveBeenCalledTimes(1)
+  })
+})