Repository navigation
refactor(pd): report decode KV handshake through the httpserver - #1737
hiworldwzj wants to merge 3 commits into
Conversation
|
Follow-up patch on top of What it fixes
Approach
Tests New unit tests cover routing by master, a handshake waiting for the connection, a slow master not blocking another, retry after a failed send, expiry, a full IPC queue failing the request (real ZMQ), and The PD unit tests ( Apply with 0001-fix-pd-queue-decode-KV-handshakes-per-PD-master.patchFrom e869c4a1a1a00c80090b1699515e1c7f8679011d Mon Sep 17 00:00:00 2001
From: sufubao <sufubao@sensetime.com>
Date: Tue, 17 Nov 2026 00:54:19 -0800
Subject: [PATCH] fix(pd): queue decode KV handshakes per PD master
The single IPC receive loop awaited websocket.send() for each handshake,
and dropped a handshake when its master connection was not up. A slow
master therefore delayed every other master's handshakes, and a
handshake that arrived during the 10s reconnect window (or before the
first connection) was lost, leaving the request waiting until timeout.
The IPC PUSH also blocked the decode transfer thread once its queue
filled.
- The IPC loop only appends to a per-master pending queue. Each
/pd_register connection drains its own queue in a send task, next to
the token and heartbeat tasks, so masters no longer wait on each
other.
- Pending handshakes survive reconnects, a failed send is put back,
and entries expire after the 180s transfer timeout, which also bounds
the queue. The queue is dropped when the master leaves discovery.
- Push.send is non-blocking and returns False when the queue is full;
decode trans then aborts the request at once instead of waiting for
the transfer timeout.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
lightllm/server/httpserver/pd_loop.py | 105 ++++++---
lightllm/server/pd_httpserver_ipc.py | 16 +-
.../decode_node_impl/decode_trans_process.py | 5 +-
.../httpserver/test_pd_kv_status_channel.py | 207 ++++++++++++++----
.../mode_backend/test_pd_transfer_progress.py | 2 +-
5 files changed, 253 insertions(+), 82 deletions(-)
diff --git a/lightllm/server/httpserver/pd_loop.py b/lightllm/server/httpserver/pd_loop.py
index f8c46ff0b..f62b515c5 100644
--- a/lightllm/server/httpserver/pd_loop.py
+++ b/lightllm/server/httpserver/pd_loop.py
@@ -9,10 +9,12 @@ import weakref
import os
import signal
import sys
+import time
import uuid
-from typing import Dict, Optional, Union, List
+from collections import deque
+from typing import Deque, Dict, Optional, Tuple, Union, List
from websockets import ClientConnection
-from lightllm.server.pd_io_struct import NodeRole, ObjType
+from lightllm.server.pd_io_struct import NodeRole, ObjType, PDUpKVStatus
from lightllm.server.httpserver.async_queue import AsyncQueue
from lightllm.utils.net_utils import get_hostname_ip, format_host_port
from lightllm.utils.log_utils import init_logger
@@ -91,7 +93,7 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O
forwarding_tokens_task = None
heartbeat_task = None
generation_tasks: Dict[int, asyncio.Task] = {}
- registered_websocket = None
+ kv_status_task = None
try:
uri = f"ws://{pd_master_obj.host_ip_port}/pd_register"
async with websockets.connect(
@@ -119,15 +121,16 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O
await websocket.send(json.dumps(regist_json))
logger.info(f"Sent registration JSON: {regist_json}")
- if manager.pd_mode.is_D():
- manager.pd_master_forwarder.register(pd_master_obj.node_id, websocket)
- registered_websocket = websocket
# 转发任务
forwarding_tokens_task = asyncio.create_task(
_up_tokens_to_pd_master(forwarding_queue, websocket, dp_index)
)
heartbeat_task = asyncio.create_task(_send_heartbeat_to_pd_master(websocket))
+ if manager.pd_mode.is_D():
+ kv_status_task = asyncio.create_task(
+ manager.pd_master_forwarder.send_loop(pd_master_obj.node_id, websocket)
+ )
group_req_id_to_event: Dict[int, asyncio.Event] = weakref.WeakValueDictionary()
# 接收 pd master 发来的请求,并推理后,将生成的token转发回pd master。
@@ -196,15 +199,18 @@ async def _pd_handle_task(manager: HttpServerManager, pd_master_obj: PD_Master_O
except asyncio.CancelledError:
# 如果任务被取消,则退出循环
logger.warning(f"pd_handle_task {pd_master_obj} cancelled")
+ if manager.pd_mode.is_D():
+ # Master 已经下线,丢掉它的待发握手。
+ manager.pd_master_forwarder.remove_master(pd_master_obj.node_id)
return
except Exception as e:
logger.error("connetion to pd_master has error")
logger.exception(str(e))
finally:
- if registered_websocket is not None:
- manager.pd_master_forwarder.unregister(pd_master_obj.node_id, registered_websocket)
- child_tasks = [task for task in (forwarding_tokens_task, heartbeat_task) if task is not None]
+ child_tasks = [
+ task for task in (forwarding_tokens_task, heartbeat_task, kv_status_task) if task is not None
+ ]
child_tasks.extend(generation_tasks.values())
for task in child_tasks:
task.cancel()
@@ -359,42 +365,81 @@ def _get_load_info(dp_index: Optional[int] = None) -> dict:
return load_info
+class _PendingKVStatus:
+ """某个 PD Master 待发送的 KV 握手。连接断开时也保留,重连后继续发送。"""
+
+ # 与 decode 传输任务的超时一致。过期后请求已经失败,握手不再有意义,同时也限制了队列长度。
+ EXPIRE_SECS = 180
+
+ def __init__(self) -> None:
+ self._items: Deque[Tuple[float, PDUpKVStatus]] = deque()
+ self._ready = asyncio.Event()
+
+ def put(self, upkv_status: PDUpKVStatus) -> None:
+ self._drop_expired()
+ self._items.append((time.time(), upkv_status))
+ self._ready.set()
+
+ def put_back(self, item: Tuple[float, PDUpKVStatus]) -> None:
+ # 发送没完成就放回队首,保持顺序。重复送达无害,PD Master 只是再 set 一次 event。
+ self._items.appendleft(item)
+ self._ready.set()
+
+ async def get(self) -> Tuple[float, PDUpKVStatus]:
+ while True:
+ self._drop_expired()
+ if self._items:
+ return self._items.popleft()
+ self._ready.clear()
+ await self._ready.wait()
+
+ def _drop_expired(self) -> None:
+ deadline = time.time() - self.EXPIRE_SECS
+ while self._items and self._items[0][0] < deadline:
+ _, upkv_status = self._items.popleft()
+ logger.warning(f"up kv status expired before pd master connection is ready, drop {upkv_status}")
+
+
class PDMasterForwarder:
"""Decode 上把本机 IPC 收到的 KV 握手转发给 PD Master。
- 每个 httpserver worker 都连着所有 PD Master,握手里带着 pd_master_node_id,
- 所以不管 IPC 落到哪个 worker,都能沿该 worker 自己到这个 PD Master 的 /pd_register 连接送达。
+ 握手里带着 pd_master_node_id,沿到这个 PD Master 的 /pd_register 连接送达。
+ IPC 循环只按 Master 入队,每条连接各自发送:一个 Master 变慢不影响其他 Master,
+ 连接还没建立或正在重连时握手先排队,连上后再发。
"""
def __init__(self) -> None:
- self._node_id_to_websocket: Dict[int, ClientConnection] = {}
-
- def register(self, node_id: int, websocket: ClientConnection) -> None:
- self._node_id_to_websocket[node_id] = websocket
-
- def unregister(self, node_id: int, websocket: ClientConnection) -> None:
- # 重连时新连接可能已经登记,只删除自己那条。
- if self._node_id_to_websocket.get(node_id) is websocket:
- del self._node_id_to_websocket[node_id]
+ self._node_id_to_pending: Dict[int, _PendingKVStatus] = {}
+
+ def _pending(self, node_id: int) -> _PendingKVStatus:
+ # 可能还没发现这个 Master 或正在重连,先排队,连上后再发。
+ return self._node_id_to_pending.setdefault(node_id, _PendingKVStatus())
+
+ def remove_master(self, node_id: int) -> None:
+ self._node_id_to_pending.pop(node_id, None)
+
+ async def send_loop(self, node_id: int, websocket: ClientConnection) -> None:
+ """随每条 /pd_register 连接运行,连接结束时被取消。"""
+ pending = self._pending(node_id)
+ while True:
+ item = await pending.get()
+ try:
+ await websocket.send(pickle.dumps((ObjType.PD_UPLOAD_KV_STATUS, item[1])))
+ except BaseException:
+ pending.put_back(item)
+ raise
async def run(self) -> None:
from lightllm.server.pd_httpserver_ipc import PDHttpserverIpc
pull = PDHttpserverIpc.Pull()
try:
- # 同一个地址只能绑一次。别的 httpserver 进程再 bind 会失败并记日志。
+ # Prefill / Decode 只有一个 httpserver worker,由它独占这个地址。
pull.bind()
logger.info("pd httpserver ipc receiver started in this httpserver worker")
while True:
- upkv_status = await pull.recv()
- try:
- websocket = self._node_id_to_websocket.get(upkv_status.pd_master_node_id)
- if websocket is None:
- logger.warning(f"no connection to pd master, drop up kv status {upkv_status}")
- continue
- await websocket.send(pickle.dumps((ObjType.PD_UPLOAD_KV_STATUS, upkv_status)))
- except Exception as e:
- logger.exception(f"upload pd kv status failed, request_id={upkv_status.group_request_id}: {str(e)}")
+ upkv_status: PDUpKVStatus = await pull.recv()
+ self._pending(upkv_status.pd_master_node_id).put(upkv_status)
except asyncio.CancelledError:
raise
except BaseException as e:
diff --git a/lightllm/server/pd_httpserver_ipc.py b/lightllm/server/pd_httpserver_ipc.py
index a3c9511dd..f7a9d3a5d 100644
--- a/lightllm/server/pd_httpserver_ipc.py
+++ b/lightllm/server/pd_httpserver_ipc.py
@@ -6,8 +6,8 @@
httpserver 先 bind,同一个地址只能绑一次,失败直接抛异常。
传输进程在 httpserver 起来之后再 connect。连接保持时,各发送方的消息都会送到,
-同一发送方内部保持顺序。send() 返回只表示进入本进程队列,对端没有确认。
-对端还没接上、连接断开或进程退出时,队列里的消息会丢。
+同一发送方内部保持顺序。send() 不阻塞,队列已满时返回 False;返回 True 只表示进入本进程队列,
+对端没有确认。对端还没接上、连接断开或进程退出时,队列里的消息会丢。
"""
import pickle
@@ -21,10 +21,16 @@ class PDHttpserverIpc:
self._context = None
self._socket = None
- def send(self, message: object) -> None:
- # 第一次发送时才 connect。返回只表示消息进了本进程队列。
+ def send(self, message: object) -> bool:
+ """第一次发送时才 connect。不阻塞调用方,队列已满时返回 False,由调用方处理。"""
+ import zmq
+
self._ensure_connected()
- self._socket.send(pickle.dumps(message))
+ try:
+ self._socket.send(pickle.dumps(message), flags=zmq.NOBLOCK)
+ return True
+ except zmq.Again:
+ return False
def _ensure_connected(self) -> None:
if self._socket is not None:
diff --git a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py
index 6757c5fbe..b2a88aed8 100644
--- a/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py
+++ b/lightllm/server/router/model_infer/mode_backend/pd/decode_node_impl/decode_trans_process.py
@@ -252,7 +252,10 @@ class _DecodeTransModule:
pd_kv_trans_params=pickle.dumps(decode_node_info),
)
# 送给本机 httpserver,由它按 pd_master_node_id 找到 PD Master 连接再转发。
- self.pd_httpserver_push.send(message=up_status)
+ if not self.pd_httpserver_push.send(message=up_status):
+ # 握手只发一次,发不出去 Prefill 永远不会来写,立刻让请求失败,不必等传输超时。
+ logger.error(f"pd httpserver ipc queue is full, fail request {task.request_id}")
+ self._abort(request_id=task.request_id, error_info="pd httpserver ipc queue is full")
@log_exception
def accept_peer_task_loop(
diff --git a/unit_tests/server/httpserver/test_pd_kv_status_channel.py b/unit_tests/server/httpserver/test_pd_kv_status_channel.py
index bb23b438b..4b27b0a3a 100644
--- a/unit_tests/server/httpserver/test_pd_kv_status_channel.py
+++ b/unit_tests/server/httpserver/test_pd_kv_status_channel.py
@@ -1,12 +1,19 @@
"""Decode trans packs the KV handshake and httpserver forwards it to the PD master."""
import asyncio
+import glob
+import os
import pickle
+import uuid
from types import SimpleNamespace
from unittest.mock import AsyncMock
+import pytest
+
+from lightllm.server.httpserver import pd_loop
from lightllm.server.httpserver.pd_loop import PDMasterForwarder
from lightllm.server.httpserver_for_pd_master.manager import HttpServerManagerForPDMaster
+from lightllm.server.pd_httpserver_ipc import PDHttpserverIpc
from lightllm.server.pd_io_struct import ObjType, PDDecodeNodeInfo, PDUpKVStatus
from lightllm.server.router.model_infer.mode_backend.pd.decode_node_impl import decode_trans_process
@@ -15,7 +22,7 @@ def test_decode_trans_packs_the_handshake_for_httpserver():
sent = []
module = decode_trans_process._DecodeTransModule.__new__(decode_trans_process._DecodeTransModule)
module.args = SimpleNamespace(pd_node_id=7)
- module.pd_httpserver_push = SimpleNamespace(send=lambda message: sent.append(pickle.dumps(message)))
+ module.pd_httpserver_push = SimpleNamespace(send=lambda message: sent.append(pickle.dumps(message)) or True)
module.transporter = SimpleNamespace(
agent_name="7_1",
agent_metadata=b"meta",
@@ -45,18 +52,31 @@ def test_decode_trans_packs_the_handshake_for_httpserver():
)
-def _run_one_ipc_message(monkeypatch, forwarder, status):
- delivered = False
+def test_decode_trans_fails_the_request_when_ipc_queue_is_full():
+ # 握手只发一次,发不出去就立刻失败,而不是等 180 秒传输超时。
+ aborted = []
+ module = decode_trans_process._DecodeTransModule.__new__(decode_trans_process._DecodeTransModule)
+ module.args = SimpleNamespace(pd_node_id=7)
+ module.pd_httpserver_push = SimpleNamespace(send=lambda message: False)
+ module.transporter = SimpleNamespace(agent_name="7_1", agent_metadata=b"", num_pages=3, local_page_mem_desc=b"")
+ module._abort = lambda request_id, error_info: aborted.append(request_id)
+
+ module._push_up_kv_status(task=SimpleNamespace(request_id=8, pd_master_node_id=100, start_kv_index=6))
+
+ assert aborted == [8]
+
+
+def _ipc_messages(monkeypatch, statuses):
+ """把 forwarder 的 IPC 接收端换成依次吐出 statuses 的假实现。"""
+ pending = list(statuses)
class Pull:
def bind(self) -> None:
return None
async def recv(self):
- nonlocal delivered
- if not delivered:
- delivered = True
- return status
+ if pending:
+ return pending.pop(0)
await asyncio.Event().wait()
def close(self) -> None:
@@ -64,62 +84,159 @@ def _run_one_ipc_message(monkeypatch, forwarder, status):
monkeypatch.setattr("lightllm.server.pd_httpserver_ipc.PDHttpserverIpc.Pull", Pull)
- async def go():
- task = asyncio.create_task(forwarder.run())
- for _ in range(50):
- if delivered:
- break
- await asyncio.sleep(0.01)
- await asyncio.sleep(0.01)
- task.cancel()
- await asyncio.gather(task, return_exceptions=True)
- asyncio.run(go())
+class _Websocket:
+ def __init__(self, blocked: bool = False, broken: bool = False) -> None:
+ self.sent = []
+ self.unblock = asyncio.Event()
+ if not blocked:
+ self.unblock.set()
+ self.broken = broken
+ async def send(self, payload) -> None:
+ await self.unblock.wait()
+ if self.broken:
+ raise ConnectionError("closed")
+ self.sent.append(pickle.loads(payload))
-def _recording_websocket():
- sent = []
- async def send(payload):
- sent.append(pickle.loads(payload))
+async def _until(predicate) -> None:
+ for _ in range(200):
+ if predicate():
+ return
+ await asyncio.sleep(0.005)
+ raise AssertionError("condition not reached")
- return SimpleNamespace(send=send), sent
+
+async def _cancel(*tasks) -> None:
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+
+def _kv(request_id: int, node_id: int) -> tuple:
+ status = PDUpKVStatus(request_id, node_id, b"metadata")
+ return status, (ObjType.PD_UPLOAD_KV_STATUS, status)
def test_httpserver_forwards_the_handshake_by_pd_master_node(monkeypatch):
# 请求可能是从别的 httpserver worker 进来的,这里只靠 pd_master_node_id 选连接。
- status = PDUpKVStatus(9, 100, b"metadata")
- master_100, sent_100 = _recording_websocket()
- master_200, sent_200 = _recording_websocket()
+ status, expected = _kv(9, 100)
+ _ipc_messages(monkeypatch, [status])
+
+ async def run():
+ forwarder = PDMasterForwarder()
+ master_100, master_200 = _Websocket(), _Websocket()
+ tasks = [
+ asyncio.create_task(forwarder.send_loop(100, master_100)),
+ asyncio.create_task(forwarder.send_loop(200, master_200)),
+ asyncio.create_task(forwarder.run()),
+ ]
+ await _until(lambda: master_100.sent)
+ await _cancel(*tasks)
+ assert master_100.sent == [expected]
+ assert master_200.sent == []
+
+ asyncio.run(run())
+
- forwarder = PDMasterForwarder()
- forwarder.register(node_id=100, websocket=master_100)
- forwarder.register(node_id=200, websocket=master_200)
+def test_handshake_waits_for_the_master_connection(monkeypatch):
+ # 接收 IPC 的 worker 到该 Master 的连接还没建立或正在重连时,握手要等连接恢复后再发。
+ status, expected = _kv(8, 100)
+ _ipc_messages(monkeypatch, [status])
+
+ async def run():
+ forwarder = PDMasterForwarder()
+ receiver = asyncio.create_task(forwarder.run())
+ await asyncio.sleep(0.02)
+
+ websocket = _Websocket()
+ sender = asyncio.create_task(forwarder.send_loop(100, websocket))
+ await _until(lambda: websocket.sent)
+ await _cancel(receiver, sender)
+ assert websocket.sent == [expected]
+
+ asyncio.run(run())
+
+
+def test_slow_master_does_not_block_other_masters(monkeypatch):
+ slow_status, _ = _kv(1, 100)
+ fast_status, fast_expected = _kv(2, 200)
+ _ipc_messages(monkeypatch, [slow_status, fast_status])
+
+ async def run():
+ forwarder = PDMasterForwarder()
+ slow, fast = _Websocket(blocked=True), _Websocket()
+ tasks = [
+ asyncio.create_task(forwarder.send_loop(100, slow)),
+ asyncio.create_task(forwarder.send_loop(200, fast)),
+ asyncio.create_task(forwarder.run()),
+ ]
+ await _until(lambda: fast.sent)
+ assert fast.sent == [fast_expected]
+ assert slow.sent == []
+ await _cancel(*tasks)
+
+ asyncio.run(run())
- _run_one_ipc_message(monkeypatch, forwarder, status)
- assert sent_100 == [(ObjType.PD_UPLOAD_KV_STATUS, status)]
- assert sent_200 == []
+def test_failed_send_is_retried_on_the_next_connection(monkeypatch):
+ status, expected = _kv(8, 100)
+ _ipc_messages(monkeypatch, [status])
-def test_httpserver_drops_the_handshake_without_master_connection(monkeypatch):
- forwarder = PDMasterForwarder()
- status = PDUpKVStatus(8, 100, b"metadata")
+ async def run():
+ forwarder = PDMasterForwarder()
+ receiver = asyncio.create_task(forwarder.run())
+ broken = asyncio.create_task(forwarder.send_loop(100, _Websocket(broken=True)))
+ with pytest.raises(ConnectionError):
+ await broken
+
+ websocket = _Websocket()
+ sender = asyncio.create_task(forwarder.send_loop(100, websocket))
+ await _until(lambda: websocket.sent)
+ await _cancel(receiver, sender)
+ assert websocket.sent == [expected]
+
+ asyncio.run(run())
- _run_one_ipc_message(monkeypatch, forwarder, status)
+def test_expired_handshake_is_dropped(monkeypatch):
+ status, _ = _kv(8, 100)
+ _ipc_messages(monkeypatch, [status])
+ monkeypatch.setattr(pd_loop._PendingKVStatus, "EXPIRE_SECS", 0.01)
-def test_stale_connection_does_not_unregister_its_replacement():
- old, _ = _recording_websocket()
- new, _ = _recording_websocket()
- forwarder = PDMasterForwarder()
- forwarder.register(node_id=100, websocket=old)
- forwarder.register(node_id=100, websocket=new)
+ async def run():
+ forwarder = PDMasterForwarder()
+ receiver = asyncio.create_task(forwarder.run())
+ await asyncio.sleep(0.05)
+
+ websocket = _Websocket()
+ sender = asyncio.create_task(forwarder.send_loop(100, websocket))
+ await asyncio.sleep(0.05)
+ await _cancel(receiver, sender)
+ assert websocket.sent == []
+
+ asyncio.run(run())
- forwarder.unregister(node_id=100, websocket=old)
- assert forwarder._node_id_to_websocket == {100: new}
- forwarder.unregister(node_id=100, websocket=new)
- assert forwarder._node_id_to_websocket == {}
+@pytest.fixture
+def ipc_name(monkeypatch):
+ name = f"test_{uuid.uuid4().hex}"
+ monkeypatch.setenv("LIGHTLLM_UNIQUE_SERVICE_NAME_ID", name)
+ yield name
+ for path in glob.glob(f"/tmp/{name}_pd_httpserver*"):
+ os.remove(path)
+
+
+def test_push_never_blocks_and_reports_a_full_queue(ipc_name):
+ push = PDHttpserverIpc.Push()
+ try:
+ results = [push.send(PDUpKVStatus(i, 100, b"x")) for i in range(2000)]
+ finally:
+ push._socket.close(linger=0)
+ push._context.term()
+ assert results[0] is True
+ assert results[-1] is False
def test_master_reports_kv_status_from_the_register_channel(monkeypatch):
diff --git a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_progress.py b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_progress.py
index d8d5d8ba2..48698f049 100644
--- a/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_progress.py
+++ b/unit_tests/server/router/model_infer/mode_backend/test_pd_transfer_progress.py
@@ -62,7 +62,7 @@ def make_module(*tasks):
local_page_mem_desc=b"decode page",
send_error_info_to_prefill_node=Mock(),
)
- module.pd_httpserver_push = SimpleNamespace(send=lambda message: None)
+ module.pd_httpserver_push = SimpleNamespace(send=lambda message: True)
for name in (
"request_page_task_queue",
"ready_page_task_queue",
--
2.34.1
|
Drop the dedicated up_kv_status process and /kv_move_status socket. Decode trans pushes the handshake to one local httpserver after the task is waiting, and that httpserver forwards it on /pd_register. Co-authored-by: Cursor <cursoragent@cursor.com>
Each PD master worker has its own node id. Decode forwards the local handshake on the /pd_register connection for that id, and prefill/decode nodes keep a single httpserver worker. Co-authored-by: Cursor <cursoragent@cursor.com>
Drop the dedicated up_kv_status process and /kv_move_status socket. Decode trans pushes the handshake to one local httpserver after the task is waiting, and that httpserver forwards it on /pd_register.