Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 102 additions & 101 deletions lightllm/server/httpserver/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,109 +849,110 @@ async def _wait_to_token_package(
group_request_id=group_request_id, reason="_wait_to_token_package check network disconnected"
)

# 只在锁内取走当前批次,yield 必须在锁外执行:慢速客户端挂起生成器时若仍持有锁,
# 会阻塞 handle_loop 对所有请求的 token 搬运,导致请求资源无法回收。
async with req_status.lock:
event.clear()
if len(req_status.out_token_info_list) == 0:
continue

for sub_req_id, out_str, metadata, finish_status in req_status.out_token_info_list:
# pd master 节点需要这个做统计信息, 所以放在元数据中返回给 pd master 节点
metadata["prompt_tokens"] = prompt_tokens

gpu_prompt_cache_len = metadata.pop("prompt_cache_len", 0)
cpu_prompt_cache_len = metadata.pop("cpu_prompt_cache_len", 0)
disk_prompt_cache_len = metadata.pop("disk_prompt_cache_len", 0)
metadata["prompt_cache_len"] = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len
sub_req_id_to_mtp_accepted_token_num[sub_req_id] = metadata.get("mtp_accepted_token_num", 0)
cur_mtp_verify_token_num = metadata.get("mtp_verify_token_num", 0)
sub_req_id_to_mtp_verify_token_num[sub_req_id] = cur_mtp_verify_token_num
cur_mtp_verify_step_num = metadata.get("mtp_verify_step_num", 0)
sub_req_id_to_mtp_verify_step_num[sub_req_id] = cur_mtp_verify_step_num

if is_first_token:
first_token_cost_ms = (time.time() - start_time) * 1000
is_first_token = False
self.first_time_costs.add(first_token_cost_ms)

out_token_counter += 1

# update inference timemark
self.latest_success_infer_time_mark.set_value(int(time.time()))

yield sub_req_id, out_str, metadata, finish_status
# 如果有子请求完成,就更新计数
if finish_status.is_finished():
unfinished_count -= 1

if unfinished_count == 0:
total_cost_time_ms = (time.time() - start_time) * 1000
mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter
self.per_token_costs.add(mean_per_token_cost_time_ms)
x_request_id = request.headers.get("X-Request-Id", "") if request is not None else ""
x_session_id = request.headers.get("X-Session-Id", "") if request is not None else ""
gpu_prompt_cache_ratio = gpu_prompt_cache_len / prompt_tokens
cpu_prompt_cache_ratio = cpu_prompt_cache_len / prompt_tokens
disk_prompt_cache_ratio = disk_prompt_cache_len / prompt_tokens
prompt_cache_len = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len
prompt_cache_ratio = prompt_cache_len / prompt_tokens
generation_throughput = out_token_counter / max(total_cost_time_ms / 1000.0, 1e-6)

mtp_accepted_token_num = sum(sub_req_id_to_mtp_accepted_token_num.values())
mtp_verify_token_num = sum(sub_req_id_to_mtp_verify_token_num.values())
mtp_total_verify_steps = sum(sub_req_id_to_mtp_verify_step_num.values())
if mtp_total_verify_steps <= 0:
mtp_total_verify_steps = out_token_counter - mtp_accepted_token_num
mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1)
mtp_avg_verify_tokens_per_step = mtp_verify_token_num / max(mtp_total_verify_steps, 1)
mtp_avg_accepted_tokens_per_step = mtp_accepted_token_num / max(mtp_total_verify_steps, 1)
format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S")
logger.debug(
f"X-Request-Id:{x_request_id} "
f"X-Session-Id:{x_session_id} start_time:{format_start_time} "
f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms "
f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} "
f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms "
f"prompt_token_num:{prompt_tokens} "
f"gpu cache hit: {gpu_prompt_cache_ratio > 0} "
f"gpu_prompt_cache_len:{gpu_prompt_cache_len} "
f"gpu_prompt_cache_ratio:{gpu_prompt_cache_ratio} "
f"cpu cache hit: {cpu_prompt_cache_len > 0} "
f"cpu_prompt_cache_len:{cpu_prompt_cache_len} "
f"cpu_prompt_cache_ratio:{cpu_prompt_cache_ratio} "
f"disk cache hit: {disk_prompt_cache_len > 0} "
f"disk_prompt_cache_len:{disk_prompt_cache_len} "
f"disk_prompt_cache_ratio:{disk_prompt_cache_ratio} "
f"mtp_accepted_token_num:{mtp_accepted_token_num} "
f"mtp_total_verify_steps:{mtp_total_verify_steps} "
f"mtp_total_verify_tokens:{mtp_verify_token_num} "
f"mtp_avg_token_per_step:{mtp_avg_token_per_step} "
f"mtp_avg_accepted_tokens_per_step:{mtp_avg_accepted_tokens_per_step} "
f"mtp_avg_verify_tokens_per_step:{mtp_avg_verify_tokens_per_step} "
)

self.metric_client.histogram_observe("lightllm_cache_length", prompt_cache_len)
self.metric_client.histogram_observe("lightllm_cache_ratio", prompt_cache_ratio)
self.metric_client.counter_inc_by("lightllm_prompt_tokens_total", prompt_tokens)
self.metric_client.counter_inc_by("lightllm_generation_tokens_total", out_token_counter)
self.metric_client.gauge_set("lightllm_cache_hit_rate", prompt_cache_ratio)
self.metric_client.gauge_set("lightllm_gen_throughput", generation_throughput)
self.metric_client.histogram_observe(
"lightllm_request_inference_duration", total_cost_time_ms / 1000.0
)
self.metric_client.histogram_observe(
"lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0
)
self.metric_client.histogram_observe(
"lightllm_request_first_token_duration", first_token_cost_ms / 1000.0
)
self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
self.metric_client.counter_inc("lightllm_request_success")
self.metric_client.histogram_observe(
"lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step
)

return
req_status.out_token_info_list.clear()
token_batch = req_status.out_token_info_list
req_status.out_token_info_list = []

for sub_req_id, out_str, metadata, finish_status in token_batch:
# pd master 节点需要这个做统计信息, 所以放在元数据中返回给 pd master 节点
metadata["prompt_tokens"] = prompt_tokens

gpu_prompt_cache_len = metadata.pop("prompt_cache_len", 0)
cpu_prompt_cache_len = metadata.pop("cpu_prompt_cache_len", 0)
disk_prompt_cache_len = metadata.pop("disk_prompt_cache_len", 0)
metadata["prompt_cache_len"] = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len
sub_req_id_to_mtp_accepted_token_num[sub_req_id] = metadata.get("mtp_accepted_token_num", 0)
cur_mtp_verify_token_num = metadata.get("mtp_verify_token_num", 0)
sub_req_id_to_mtp_verify_token_num[sub_req_id] = cur_mtp_verify_token_num
cur_mtp_verify_step_num = metadata.get("mtp_verify_step_num", 0)
sub_req_id_to_mtp_verify_step_num[sub_req_id] = cur_mtp_verify_step_num

if is_first_token:
first_token_cost_ms = (time.time() - start_time) * 1000
is_first_token = False
self.first_time_costs.add(first_token_cost_ms)

out_token_counter += 1

# update inference timemark
self.latest_success_infer_time_mark.set_value(int(time.time()))

yield sub_req_id, out_str, metadata, finish_status
# 如果有子请求完成,就更新计数
if finish_status.is_finished():
unfinished_count -= 1

if unfinished_count == 0:
total_cost_time_ms = (time.time() - start_time) * 1000
mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter
self.per_token_costs.add(mean_per_token_cost_time_ms)
x_request_id = request.headers.get("X-Request-Id", "") if request is not None else ""
x_session_id = request.headers.get("X-Session-Id", "") if request is not None else ""
gpu_prompt_cache_ratio = gpu_prompt_cache_len / prompt_tokens
cpu_prompt_cache_ratio = cpu_prompt_cache_len / prompt_tokens
disk_prompt_cache_ratio = disk_prompt_cache_len / prompt_tokens
prompt_cache_len = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len
prompt_cache_ratio = prompt_cache_len / prompt_tokens
generation_throughput = out_token_counter / max(total_cost_time_ms / 1000.0, 1e-6)

mtp_accepted_token_num = sum(sub_req_id_to_mtp_accepted_token_num.values())
mtp_verify_token_num = sum(sub_req_id_to_mtp_verify_token_num.values())
mtp_total_verify_steps = sum(sub_req_id_to_mtp_verify_step_num.values())
if mtp_total_verify_steps <= 0:
mtp_total_verify_steps = out_token_counter - mtp_accepted_token_num
mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1)
mtp_avg_verify_tokens_per_step = mtp_verify_token_num / max(mtp_total_verify_steps, 1)
mtp_avg_accepted_tokens_per_step = mtp_accepted_token_num / max(mtp_total_verify_steps, 1)
format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S")
logger.debug(
f"X-Request-Id:{x_request_id} "
f"X-Session-Id:{x_session_id} start_time:{format_start_time} "
f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms "
f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} "
f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms "
f"prompt_token_num:{prompt_tokens} "
f"gpu cache hit: {gpu_prompt_cache_ratio > 0} "
f"gpu_prompt_cache_len:{gpu_prompt_cache_len} "
f"gpu_prompt_cache_ratio:{gpu_prompt_cache_ratio} "
f"cpu cache hit: {cpu_prompt_cache_len > 0} "
f"cpu_prompt_cache_len:{cpu_prompt_cache_len} "
f"cpu_prompt_cache_ratio:{cpu_prompt_cache_ratio} "
f"disk cache hit: {disk_prompt_cache_len > 0} "
f"disk_prompt_cache_len:{disk_prompt_cache_len} "
f"disk_prompt_cache_ratio:{disk_prompt_cache_ratio} "
f"mtp_accepted_token_num:{mtp_accepted_token_num} "
f"mtp_total_verify_steps:{mtp_total_verify_steps} "
f"mtp_total_verify_tokens:{mtp_verify_token_num} "
f"mtp_avg_token_per_step:{mtp_avg_token_per_step} "
f"mtp_avg_accepted_tokens_per_step:{mtp_avg_accepted_tokens_per_step} "
f"mtp_avg_verify_tokens_per_step:{mtp_avg_verify_tokens_per_step} "
)

self.metric_client.histogram_observe("lightllm_cache_length", prompt_cache_len)
self.metric_client.histogram_observe("lightllm_cache_ratio", prompt_cache_ratio)
self.metric_client.counter_inc_by("lightllm_prompt_tokens_total", prompt_tokens)
self.metric_client.counter_inc_by("lightllm_generation_tokens_total", out_token_counter)
self.metric_client.gauge_set("lightllm_cache_hit_rate", prompt_cache_ratio)
self.metric_client.gauge_set("lightllm_gen_throughput", generation_throughput)
self.metric_client.histogram_observe(
"lightllm_request_inference_duration", total_cost_time_ms / 1000.0
)
self.metric_client.histogram_observe(
"lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0
)
self.metric_client.histogram_observe(
"lightllm_request_first_token_duration", first_token_cost_ms / 1000.0
)
self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter)
self.metric_client.counter_inc("lightllm_request_success")
self.metric_client.histogram_observe(
"lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step
)

return
return

async def stop_request(self, request_id: int):
Expand Down
42 changes: 42 additions & 0 deletions unit_tests/server/httpserver/test_running_request_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -587,3 +587,45 @@ def test_real_supported_max_req_total_len_reserves_mtp_overlap_tokens():
manager.args = SimpleNamespace(mtp_step=2, page_size=4)

assert manager.get_real_supported_max_req_total_len() == 951


def test_wait_to_token_package_releases_lock_while_consumer_is_suspended():
# 慢速流式客户端会让生成器停在 yield 处;此时若仍持有 req_status.lock,
# handle_loop 无法继续搬运 token,请求资源也就无法回收。
async def run():
manager = _make_manager(NodeRole.NORMAL)
manager.first_time_costs = MagicMock()
manager.per_token_costs = MagicMock()
manager.latest_success_infer_time_mark = _ValueMark()
req = SimpleNamespace(infer_start_time=1.0)
req_status = _req_status([req])
sampling_params = _sampling_params()
sampling_params.best_of = 1

def token(text, status=FinishStatus.NO_FINISH):
return (123, text, {"prompt_cache_len": 0}, FinishStatus(status))

req_status.out_token_info_list.extend([token("a"), token("b")])
req_status.event.set()
output_generator = manager._wait_to_token_package(
start_time=0,
prompt_ids=[1, 2, 3],
group_request_id=123,
sampling_params=sampling_params,
req_status=req_status,
request=None,
)

first = await output_generator.__anext__()
assert not req_status.lock.locked()

async def produce():
async with req_status.lock:
req_status.out_token_info_list.append(token("c", FinishStatus.FINISHED_STOP))
req_status.event.set()

await asyncio.wait_for(produce(), timeout=1)
rest = [item async for item in output_generator]
assert [first[1]] + [item[1] for item in rest] == ["a", "b", "c"]

asyncio.run(run())
Loading