diff --git a/docs/sentinel.md b/docs/sentinel.md index 1e1ab9aa34e..aa1f06cd0c1 100644 --- a/docs/sentinel.md +++ b/docs/sentinel.md @@ -74,7 +74,7 @@ const sentinel = await createSentinel({ | Property | Default | Description | |----------------------------|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | name | | The sentinel identifier for a particular database cluster | -| sentinelRootNodes | | An array of root nodes that are part of the sentinel cluster, which will be used to get the topology. Each element in the array is a client configuration object. There is no need to specify every node in the cluster: 3 should be enough to reliably connect and obtain the sentinel configuration from the server | +| sentinelRootNodes | | An array of root nodes that are part of the sentinel cluster, which will be used to get the topology. Each element in the array is a client configuration object. There is no need to specify every node in the cluster: 3 should be enough to reliably connect and obtain the sentinel configuration from the server. These nodes are treated as seeds and are always kept as reconnection candidates — see [Reconnecting after an outage](#reconnecting-after-an-outage). | | maxCommandRediscovers | `16` | The maximum number of times a command will retry due to topology changes. | | nodeClientOptions | | The configuration values for every node in the cluster. Use this for example when specifying an ACL user to connect with | | sentinelClientOptions | | The configuration values for every sentinel in the cluster. Use this for example when specifying an ACL user to connect with | @@ -85,6 +85,17 @@ const sentinel = await createSentinel({ | passthroughClientErrorEvents | `false` | When `true`, error events from client instances inside the sentinel will be propagated to the sentinel instance. This allows handling all client errors through a single error handler on the sentinel instance. | | reserveClient | `false` | When `true`, one client will be reserved for the sentinel object. When `false`, the sentinel object will wait for the first available client from the pool. | +## Reconnecting after an outage + +As the client learns the sentinel topology it discovers additional sentinel nodes (reported by the sentinels as IP addresses). The nodes you pass in `sentinelRootNodes` are kept as **seeds**: they are always retained as reconnection candidates and are tried first, alongside the discovered nodes. This matters after an outage where the whole sentinel set restarts. + +Whether the client can recover depends on what the seeds resolve to: + +- **Hostname seeds** (e.g. a DNS name or a Kubernetes service) re-resolve on every reconnect attempt, so the client follows the sentinels to their new addresses even if every IP changed. This is the most robust configuration and is recommended for environments with ephemeral addressing (Kubernetes, cloud autoscaling, DHCP). +- **IP-literal seeds** recover only if the sentinels come back at the same addresses (static IP / bare-metal / fixed-IP container setups). If every sentinel restarts on a new IP and the seeds are IP literals, the client has no resolvable address left to reconnect to — there is no information from which to discover the new addresses. Use hostnames to avoid this. + +A stale seed that never comes back is harmless: the client fails to connect to it and moves on to the next candidate. + ## PubSub It supports PubSub via the normal mechanisms, including migrating the listeners if the node they are connected to goes down. diff --git a/packages/client/lib/sentinel/index.spec.ts b/packages/client/lib/sentinel/index.spec.ts index a54cbf5c076..0625f9100b9 100644 --- a/packages/client/lib/sentinel/index.spec.ts +++ b/packages/client/lib/sentinel/index.spec.ts @@ -6,6 +6,7 @@ import { ScanIteratorInterruptedError, WatchError } from "../errors"; import { RedisSentinelConfig, SentinelFramework } from "./test-util"; import { RedisSentinelEvent, RedisSentinelType, RedisSentinelClientType, RedisNode } from "./types"; import RedisSentinel from "./index"; +import { mergeSentinelNodes } from "./utils"; import { RedisArgument, RedisModules, RedisFunctions, RedisScripts, RespVersions, TypeMapping } from '../RESP/types'; import { promisify } from 'node:util'; import { exec } from 'node:child_process'; @@ -131,6 +132,76 @@ describe('RedisSentinel', () => { assert.equal(duplicated.commandOptions?.timeout, overrideTimeout); }); + describe('mergeSentinelNodes (issue #3237)', () => { + // Regression: transform() used to replace sentinelRootNodes with the + // discovered list alone, dropping hostname-based seeds. After a full outage + // where sentinels restart with new IPs, the client then had no resolvable + // address left to reconnect to. mergeSentinelNodes keeps the configured + // hostname seeds available alongside discovered nodes. + + it('keeps hostname seeds when sentinel reports only new IPs', () => { + const seeds = [ + { host: 'redis-sentinel-0.svc.local', port: 26379 }, + { host: 'redis-sentinel-1.svc.local', port: 26380 }, + ]; + const discovered = [ + { host: '10.0.0.1', port: 26379 }, + { host: '10.0.0.2', port: 26380 }, + ]; + + const merged = mergeSentinelNodes(seeds, discovered); + + assert.deepEqual(merged, [...seeds, ...discovered]); + }); + + it('places seeds first, then appends discovered nodes', () => { + const seeds = [{ host: 'seed.local', port: 26379 }]; + const discovered = [ + { host: '10.0.0.1', port: 26379 }, + { host: '10.0.0.2', port: 26380 }, + ]; + + const merged = mergeSentinelNodes(seeds, discovered); + + assert.equal(merged[0].host, 'seed.local'); + assert.deepEqual(merged.slice(1), discovered); + }); + + it('dedupes by host:port across seeds and discovered nodes', () => { + const seeds = [{ host: '10.0.0.1', port: 26379 }]; + const discovered = [ + { host: '10.0.0.1', port: 26379 }, // duplicate of seed + { host: '10.0.0.2', port: 26380 }, + ]; + + const merged = mergeSentinelNodes(seeds, discovered); + + assert.equal(merged.length, 2); + assert.deepEqual(merged, [ + { host: '10.0.0.1', port: 26379 }, + { host: '10.0.0.2', port: 26380 }, + ]); + }); + + it('treats same host with different ports as distinct nodes', () => { + const seeds = [{ host: '10.0.0.1', port: 26379 }]; + const discovered = [{ host: '10.0.0.1', port: 26380 }]; + + const merged = mergeSentinelNodes(seeds, discovered); + + assert.equal(merged.length, 2); + }); + + it('returns just the seeds when nothing is discovered', () => { + const seeds = [ + { host: 'seed-0.local', port: 26379 }, + { host: 'seed-1.local', port: 26380 }, + ]; + + assert.deepEqual(mergeSentinelNodes(seeds, []), seeds); + }); + }); + it('should not have HOTKEYS commands (requires session affinity)', () => { // HOTKEYS commands require session affinity and are only available on standalone clients const sentinel = RedisSentinel.create({ diff --git a/packages/client/lib/sentinel/index.ts b/packages/client/lib/sentinel/index.ts index 50acdd0d043..da13450dc76 100644 --- a/packages/client/lib/sentinel/index.ts +++ b/packages/client/lib/sentinel/index.ts @@ -6,7 +6,7 @@ import { CommandOptions } from '../client/commands-queue'; import { attachConfig } from '../commander'; import { NON_STICKY_COMMANDS } from '../commands'; import { ClientErrorEvent, NamespaceProxySentinel, NamespaceProxySentinelClient, NodeAddressMap, ProxySentinel, ProxySentinelClient, RedisNode, RedisSentinelClientType, RedisSentinelEvent, RedisSentinelOptions, RedisSentinelType, SentinelCommander } from './types'; -import { clientSocketToNode, createCommand, createFunctionCommand, createModuleCommand, createNodeList, createScriptCommand, getMappedNode, parseNode } from './utils'; +import { clientSocketToNode, createCommand, createFunctionCommand, createModuleCommand, createNodeList, createScriptCommand, getMappedNode, mergeSentinelNodes, parseNode } from './utils'; import { RedisMultiQueuedCommand } from '../multi-command'; import RedisSentinelMultiCommand, { RedisSentinelMultiCommandType } from './multi-commands'; import { PubSubListener } from '../client/pub-sub'; @@ -810,6 +810,8 @@ export class RedisSentinelInternal< this.#RESP = options.RESP; this.#sentinelSeedNodes = Array.from(options.sentinelRootNodes); + // Initial root nodes start as a copy of the seed nodes; transform() later + // merges discovered nodes on top while preserving these seeds. this.#sentinelRootNodes = Array.from(this.#sentinelSeedNodes); this.#maxCommandRediscovers = options.maxCommandRediscovers ?? 16; this.#masterPoolSize = options.masterPoolSize ?? 1; @@ -1519,11 +1521,13 @@ export class RedisSentinelInternal< } } - if (this.#sentinelNodeListKey(analyzed.sentinelList) !== this.#sentinelNodeListKey(this.#sentinelRootNodes)) { - this.#sentinelRootNodes = analyzed.sentinelList; + const mergedSentinelList = mergeSentinelNodes(this.#sentinelSeedNodes, analyzed.sentinelList); + + if (this.#sentinelNodeListKey(mergedSentinelList) !== this.#sentinelNodeListKey(this.#sentinelRootNodes)) { + this.#sentinelRootNodes = mergedSentinelList; const event: RedisSentinelEvent = { type: "SENTINE_LIST_CHANGE", - size: analyzed.sentinelList.length + size: mergedSentinelList.length } this.emit('topology-change', event); } diff --git a/packages/client/lib/sentinel/utils.ts b/packages/client/lib/sentinel/utils.ts index c2024497a2b..ab475919398 100644 --- a/packages/client/lib/sentinel/utils.ts +++ b/packages/client/lib/sentinel/utils.ts @@ -15,7 +15,7 @@ export function parseNode(node: Record): RedisNode | undefined{ } export function createNodeList(nodes: UnwrapReply>>) { - var nodeList: Array = []; + const nodeList: Array = []; for (const nodeData of nodes) { const node = parseNode(nodeData) @@ -28,6 +28,32 @@ export function createNodeList(nodes: UnwrapReply, + discoveredNodes: Array +): Array { + const seen = new Set(); + const merged: Array = []; + + for (const node of [...seedNodes, ...discoveredNodes]) { + const key = `${node.host}:${node.port}`; + if (!seen.has(key)) { + // Clone so the working root-nodes list never aliases the frozen seed + // node objects (or the discovered ones). + merged.push({ host: node.host, port: node.port }); + seen.add(key); + } + } + + return merged; +} + export function clientSocketToNode(socket: RedisSocketOptions): RedisNode { const s = socket as RedisTcpSocketOptions;