From 20c74e1ae2901f8b444463fe248a570fd9027bea Mon Sep 17 00:00:00 2001 From: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:28:17 -0700 Subject: [PATCH] [https://nvbugs/6462303][chore] Unwaive KV cache v2 scheduler tests with executor loop tracing test_chunked_prefill (nvbugs 6428002) and test_eviction_with_block_reuse (nvbugs 6462303) are waived as flaky. Both only ever report "Test terminated unexpectedly" in CI, which gives no indication of where the session died, and 50 full-stage reruns of DGX_B200-PyTorch on current main did not reproduce either failure locally. Unwaive both and opt them into TLLM_TRACE_EXECUTOR_LOOP so the next CI occurrence records exactly where the executor loop stopped. Tracing is applied through a fixture on these two tests only, since the line trace is expensive (~1.9M stdout lines / ~157 MB per traced test) -- monkeypatch restores the environment afterwards, and the MPI pool cache already retires pools whose spawn environment no longer matches, so no other test in the session is traced. Signed-off-by: Yi Zhang <187001205+yizhang-nv@users.noreply.github.com> --- .../kv_cache/test_kv_cache_v2_scheduler.py | 25 +++++++++++++++++-- tests/integration/test_lists/waives.txt | 2 -- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py index 74a03c3727b7..40821ba12948 100644 --- a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py +++ b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py @@ -119,6 +119,25 @@ _EVICT_MAX_BATCH_SIZE = 12 +@pytest.fixture +def trace_executor_loop(monkeypatch): + """Line-trace the PyExecutor event loop to stdout for one test only. + + Both tests using this fixture are unwaived flaky failures (nvbugs 6428002 + and 6462303) whose only symptom in CI is "Test terminated unexpectedly" -- + the session dies with no indication of where. ``TLLM_TRACE_EXECUTOR_LOOP`` + makes ``trace_func`` (tensorrt_llm/_utils.py) print every executed line of + the event loop, so the last line before the process dies pinpoints the hang. + + The tracing overhead is very large, so this is opted into per test rather + than set for the module or the stage. ``monkeypatch`` restores the + environment afterwards, keeping the rest of the session unaffected; the + value is read when the executor starts, i.e. after ``LLM()`` is built here, + and is inherited by the spawned MPI workers. + """ + monkeypatch.setenv("TLLM_TRACE_EXECUTOR_LOOP", "ALL") + + def _assert_all_completed(outputs, expected_count=None): """Assert all outputs have non-empty generated text.""" if expected_count is not None: @@ -247,7 +266,8 @@ def test_token_budget_limited(self): self._compare(SHORT_PROMPTS, max_num_tokens=64) # Chunked prefill — V2 matches V1 - def test_chunked_prefill(self): + # Traced while nvbugs 6428002 ("Test terminated unexpectedly") is open. + def test_chunked_prefill(self, trace_executor_loop): self._compare([LONG_PROMPT], max_tokens=64, enable_chunked_prefill=True, max_num_tokens=128) # Chunked prefill multi-request — V2 matches V1 @@ -319,7 +339,8 @@ def test_chunked_prefill_with_eviction(self): ) # Eviction + block reuse — V2 matches V1 - def test_eviction_with_block_reuse(self): + # Traced while nvbugs 6462303 ("Test terminated unexpectedly") is open. + def test_eviction_with_block_reuse(self, trace_executor_loop): _run_eviction_test( self.MODEL_PATH, EVICTION_PROMPTS_SHORT, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index da99fdb7dac5..c1e71ac99caa 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -357,8 +357,6 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:in full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074) full:sm100/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (megatron-core 0.8 is not supported in python 3.12) full:sm100/unittest/bindings SKIP (Disable for Blackwell) -kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill SKIP (https://nvbugs/6428002) -kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse SKIP (https://nvbugs/6462303) llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-bart-large-cnn] SKIP (https://nvbugs/6463812) llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6463829) llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6463829)