diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 6c66f7db53..b3404a5c52 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -849,109 +849,110 @@ async def _wait_to_token_package( group_request_id=group_request_id, reason="_wait_to_token_package check network disconnected" ) + # 只在锁内取走当前批次,yield 必须在锁外执行:慢速客户端挂起生成器时若仍持有锁, + # 会阻塞 handle_loop 对所有请求的 token 搬运,导致请求资源无法回收。 async with req_status.lock: event.clear() - if len(req_status.out_token_info_list) == 0: - continue - - for sub_req_id, out_str, metadata, finish_status in req_status.out_token_info_list: - # pd master 节点需要这个做统计信息, 所以放在元数据中返回给 pd master 节点 - metadata["prompt_tokens"] = prompt_tokens - - gpu_prompt_cache_len = metadata.pop("prompt_cache_len", 0) - cpu_prompt_cache_len = metadata.pop("cpu_prompt_cache_len", 0) - disk_prompt_cache_len = metadata.pop("disk_prompt_cache_len", 0) - metadata["prompt_cache_len"] = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len - sub_req_id_to_mtp_accepted_token_num[sub_req_id] = metadata.get("mtp_accepted_token_num", 0) - cur_mtp_verify_token_num = metadata.get("mtp_verify_token_num", 0) - sub_req_id_to_mtp_verify_token_num[sub_req_id] = cur_mtp_verify_token_num - cur_mtp_verify_step_num = metadata.get("mtp_verify_step_num", 0) - sub_req_id_to_mtp_verify_step_num[sub_req_id] = cur_mtp_verify_step_num - - if is_first_token: - first_token_cost_ms = (time.time() - start_time) * 1000 - is_first_token = False - self.first_time_costs.add(first_token_cost_ms) - - out_token_counter += 1 - - # update inference timemark - self.latest_success_infer_time_mark.set_value(int(time.time())) - - yield sub_req_id, out_str, metadata, finish_status - # 如果有子请求完成,就更新计数 - if finish_status.is_finished(): - unfinished_count -= 1 - - if unfinished_count == 0: - total_cost_time_ms = (time.time() - start_time) * 1000 - mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter - self.per_token_costs.add(mean_per_token_cost_time_ms) - x_request_id = request.headers.get("X-Request-Id", "") if request is not None else "" - x_session_id = request.headers.get("X-Session-Id", "") if request is not None else "" - gpu_prompt_cache_ratio = gpu_prompt_cache_len / prompt_tokens - cpu_prompt_cache_ratio = cpu_prompt_cache_len / prompt_tokens - disk_prompt_cache_ratio = disk_prompt_cache_len / prompt_tokens - prompt_cache_len = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len - prompt_cache_ratio = prompt_cache_len / prompt_tokens - generation_throughput = out_token_counter / max(total_cost_time_ms / 1000.0, 1e-6) - - mtp_accepted_token_num = sum(sub_req_id_to_mtp_accepted_token_num.values()) - mtp_verify_token_num = sum(sub_req_id_to_mtp_verify_token_num.values()) - mtp_total_verify_steps = sum(sub_req_id_to_mtp_verify_step_num.values()) - if mtp_total_verify_steps <= 0: - mtp_total_verify_steps = out_token_counter - mtp_accepted_token_num - mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1) - mtp_avg_verify_tokens_per_step = mtp_verify_token_num / max(mtp_total_verify_steps, 1) - mtp_avg_accepted_tokens_per_step = mtp_accepted_token_num / max(mtp_total_verify_steps, 1) - format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S") - logger.debug( - f"X-Request-Id:{x_request_id} " - f"X-Session-Id:{x_session_id} start_time:{format_start_time} " - f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms " - f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} " - f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms " - f"prompt_token_num:{prompt_tokens} " - f"gpu cache hit: {gpu_prompt_cache_ratio > 0} " - f"gpu_prompt_cache_len:{gpu_prompt_cache_len} " - f"gpu_prompt_cache_ratio:{gpu_prompt_cache_ratio} " - f"cpu cache hit: {cpu_prompt_cache_len > 0} " - f"cpu_prompt_cache_len:{cpu_prompt_cache_len} " - f"cpu_prompt_cache_ratio:{cpu_prompt_cache_ratio} " - f"disk cache hit: {disk_prompt_cache_len > 0} " - f"disk_prompt_cache_len:{disk_prompt_cache_len} " - f"disk_prompt_cache_ratio:{disk_prompt_cache_ratio} " - f"mtp_accepted_token_num:{mtp_accepted_token_num} " - f"mtp_total_verify_steps:{mtp_total_verify_steps} " - f"mtp_total_verify_tokens:{mtp_verify_token_num} " - f"mtp_avg_token_per_step:{mtp_avg_token_per_step} " - f"mtp_avg_accepted_tokens_per_step:{mtp_avg_accepted_tokens_per_step} " - f"mtp_avg_verify_tokens_per_step:{mtp_avg_verify_tokens_per_step} " - ) - - self.metric_client.histogram_observe("lightllm_cache_length", prompt_cache_len) - self.metric_client.histogram_observe("lightllm_cache_ratio", prompt_cache_ratio) - self.metric_client.counter_inc_by("lightllm_prompt_tokens_total", prompt_tokens) - self.metric_client.counter_inc_by("lightllm_generation_tokens_total", out_token_counter) - self.metric_client.gauge_set("lightllm_cache_hit_rate", prompt_cache_ratio) - self.metric_client.gauge_set("lightllm_gen_throughput", generation_throughput) - self.metric_client.histogram_observe( - "lightllm_request_inference_duration", total_cost_time_ms / 1000.0 - ) - self.metric_client.histogram_observe( - "lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0 - ) - self.metric_client.histogram_observe( - "lightllm_request_first_token_duration", first_token_cost_ms / 1000.0 - ) - self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter) - self.metric_client.counter_inc("lightllm_request_success") - self.metric_client.histogram_observe( - "lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step - ) - - return - req_status.out_token_info_list.clear() + token_batch = req_status.out_token_info_list + req_status.out_token_info_list = [] + + for sub_req_id, out_str, metadata, finish_status in token_batch: + # pd master 节点需要这个做统计信息, 所以放在元数据中返回给 pd master 节点 + metadata["prompt_tokens"] = prompt_tokens + + gpu_prompt_cache_len = metadata.pop("prompt_cache_len", 0) + cpu_prompt_cache_len = metadata.pop("cpu_prompt_cache_len", 0) + disk_prompt_cache_len = metadata.pop("disk_prompt_cache_len", 0) + metadata["prompt_cache_len"] = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len + sub_req_id_to_mtp_accepted_token_num[sub_req_id] = metadata.get("mtp_accepted_token_num", 0) + cur_mtp_verify_token_num = metadata.get("mtp_verify_token_num", 0) + sub_req_id_to_mtp_verify_token_num[sub_req_id] = cur_mtp_verify_token_num + cur_mtp_verify_step_num = metadata.get("mtp_verify_step_num", 0) + sub_req_id_to_mtp_verify_step_num[sub_req_id] = cur_mtp_verify_step_num + + if is_first_token: + first_token_cost_ms = (time.time() - start_time) * 1000 + is_first_token = False + self.first_time_costs.add(first_token_cost_ms) + + out_token_counter += 1 + + # update inference timemark + self.latest_success_infer_time_mark.set_value(int(time.time())) + + yield sub_req_id, out_str, metadata, finish_status + # 如果有子请求完成,就更新计数 + if finish_status.is_finished(): + unfinished_count -= 1 + + if unfinished_count == 0: + total_cost_time_ms = (time.time() - start_time) * 1000 + mean_per_token_cost_time_ms = (total_cost_time_ms - first_token_cost_ms) / out_token_counter + self.per_token_costs.add(mean_per_token_cost_time_ms) + x_request_id = request.headers.get("X-Request-Id", "") if request is not None else "" + x_session_id = request.headers.get("X-Session-Id", "") if request is not None else "" + gpu_prompt_cache_ratio = gpu_prompt_cache_len / prompt_tokens + cpu_prompt_cache_ratio = cpu_prompt_cache_len / prompt_tokens + disk_prompt_cache_ratio = disk_prompt_cache_len / prompt_tokens + prompt_cache_len = gpu_prompt_cache_len + cpu_prompt_cache_len + disk_prompt_cache_len + prompt_cache_ratio = prompt_cache_len / prompt_tokens + generation_throughput = out_token_counter / max(total_cost_time_ms / 1000.0, 1e-6) + + mtp_accepted_token_num = sum(sub_req_id_to_mtp_accepted_token_num.values()) + mtp_verify_token_num = sum(sub_req_id_to_mtp_verify_token_num.values()) + mtp_total_verify_steps = sum(sub_req_id_to_mtp_verify_step_num.values()) + if mtp_total_verify_steps <= 0: + mtp_total_verify_steps = out_token_counter - mtp_accepted_token_num + mtp_avg_token_per_step = out_token_counter / max(mtp_total_verify_steps, 1) + mtp_avg_verify_tokens_per_step = mtp_verify_token_num / max(mtp_total_verify_steps, 1) + mtp_avg_accepted_tokens_per_step = mtp_accepted_token_num / max(mtp_total_verify_steps, 1) + format_start_time = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S") + logger.debug( + f"X-Request-Id:{x_request_id} " + f"X-Session-Id:{x_session_id} start_time:{format_start_time} " + f"lightllm_req_id:{group_request_id} first_token_cost:{first_token_cost_ms}ms " + f"total_cost_time:{total_cost_time_ms}ms,out_token_counter:{out_token_counter} " + f"mean_per_token_cost_time: {mean_per_token_cost_time_ms}ms " + f"prompt_token_num:{prompt_tokens} " + f"gpu cache hit: {gpu_prompt_cache_ratio > 0} " + f"gpu_prompt_cache_len:{gpu_prompt_cache_len} " + f"gpu_prompt_cache_ratio:{gpu_prompt_cache_ratio} " + f"cpu cache hit: {cpu_prompt_cache_len > 0} " + f"cpu_prompt_cache_len:{cpu_prompt_cache_len} " + f"cpu_prompt_cache_ratio:{cpu_prompt_cache_ratio} " + f"disk cache hit: {disk_prompt_cache_len > 0} " + f"disk_prompt_cache_len:{disk_prompt_cache_len} " + f"disk_prompt_cache_ratio:{disk_prompt_cache_ratio} " + f"mtp_accepted_token_num:{mtp_accepted_token_num} " + f"mtp_total_verify_steps:{mtp_total_verify_steps} " + f"mtp_total_verify_tokens:{mtp_verify_token_num} " + f"mtp_avg_token_per_step:{mtp_avg_token_per_step} " + f"mtp_avg_accepted_tokens_per_step:{mtp_avg_accepted_tokens_per_step} " + f"mtp_avg_verify_tokens_per_step:{mtp_avg_verify_tokens_per_step} " + ) + + self.metric_client.histogram_observe("lightllm_cache_length", prompt_cache_len) + self.metric_client.histogram_observe("lightllm_cache_ratio", prompt_cache_ratio) + self.metric_client.counter_inc_by("lightllm_prompt_tokens_total", prompt_tokens) + self.metric_client.counter_inc_by("lightllm_generation_tokens_total", out_token_counter) + self.metric_client.gauge_set("lightllm_cache_hit_rate", prompt_cache_ratio) + self.metric_client.gauge_set("lightllm_gen_throughput", generation_throughput) + self.metric_client.histogram_observe( + "lightllm_request_inference_duration", total_cost_time_ms / 1000.0 + ) + self.metric_client.histogram_observe( + "lightllm_request_mean_time_per_token_duration", mean_per_token_cost_time_ms / 1000.0 + ) + self.metric_client.histogram_observe( + "lightllm_request_first_token_duration", first_token_cost_ms / 1000.0 + ) + self.metric_client.histogram_observe("lightllm_request_generated_tokens", out_token_counter) + self.metric_client.counter_inc("lightllm_request_success") + self.metric_client.histogram_observe( + "lightllm_request_mtp_avg_token_per_step", mtp_avg_token_per_step + ) + + return return async def stop_request(self, request_id: int): diff --git a/unit_tests/server/httpserver/test_running_request_lifecycle.py b/unit_tests/server/httpserver/test_running_request_lifecycle.py index 6e7ec26677..8ab1319b3d 100644 --- a/unit_tests/server/httpserver/test_running_request_lifecycle.py +++ b/unit_tests/server/httpserver/test_running_request_lifecycle.py @@ -587,3 +587,45 @@ def test_real_supported_max_req_total_len_reserves_mtp_overlap_tokens(): manager.args = SimpleNamespace(mtp_step=2, page_size=4) assert manager.get_real_supported_max_req_total_len() == 951 + + +def test_wait_to_token_package_releases_lock_while_consumer_is_suspended(): + # 慢速流式客户端会让生成器停在 yield 处;此时若仍持有 req_status.lock, + # handle_loop 无法继续搬运 token,请求资源也就无法回收。 + async def run(): + manager = _make_manager(NodeRole.NORMAL) + manager.first_time_costs = MagicMock() + manager.per_token_costs = MagicMock() + manager.latest_success_infer_time_mark = _ValueMark() + req = SimpleNamespace(infer_start_time=1.0) + req_status = _req_status([req]) + sampling_params = _sampling_params() + sampling_params.best_of = 1 + + def token(text, status=FinishStatus.NO_FINISH): + return (123, text, {"prompt_cache_len": 0}, FinishStatus(status)) + + req_status.out_token_info_list.extend([token("a"), token("b")]) + req_status.event.set() + output_generator = manager._wait_to_token_package( + start_time=0, + prompt_ids=[1, 2, 3], + group_request_id=123, + sampling_params=sampling_params, + req_status=req_status, + request=None, + ) + + first = await output_generator.__anext__() + assert not req_status.lock.locked() + + async def produce(): + async with req_status.lock: + req_status.out_token_info_list.append(token("c", FinishStatus.FINISHED_STOP)) + req_status.event.set() + + await asyncio.wait_for(produce(), timeout=1) + rest = [item async for item in output_generator] + assert [first[1]] + [item[1] for item in rest] == ["a", "b", "c"] + + asyncio.run(run())