[https://nvbugs/6462303][chore] Unwaive KV cache v2 scheduler tests with executor loop tracing

tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py

@@ -119,6 +119,25 @@
 _EVICT_MAX_BATCH_SIZE = 12
 
 
+@pytest.fixture
+def trace_executor_loop(monkeypatch):
+    """Line-trace the PyExecutor event loop to stdout for one test only.
+
+    Both tests using this fixture are unwaived flaky failures (nvbugs 6428002
+    and 6462303) whose only symptom in CI is "Test terminated unexpectedly" --
+    the session dies with no indication of where. ``TLLM_TRACE_EXECUTOR_LOOP``
+    makes ``trace_func`` (tensorrt_llm/_utils.py) print every executed line of
+    the event loop, so the last line before the process dies pinpoints the hang.
+
+    The tracing overhead is very large, so this is opted into per test rather
+    than set for the module or the stage. ``monkeypatch`` restores the
+    environment afterwards, keeping the rest of the session unaffected; the
+    value is read when the executor starts, i.e. after ``LLM()`` is built here,
+    and is inherited by the spawned MPI workers.
+    """
+    monkeypatch.setenv("TLLM_TRACE_EXECUTOR_LOOP", "ALL")
+
+
 def _assert_all_completed(outputs, expected_count=None):
     """Assert all outputs have non-empty generated text."""
     if expected_count is not None:
@@ -247,7 +266,8 @@ def test_token_budget_limited(self):
         self._compare(SHORT_PROMPTS, max_num_tokens=64)
 
     # Chunked prefill — V2 matches V1
-    def test_chunked_prefill(self):
+    # Traced while nvbugs 6428002 ("Test terminated unexpectedly") is open.
+    def test_chunked_prefill(self, trace_executor_loop):
         self._compare([LONG_PROMPT], max_tokens=64, enable_chunked_prefill=True, max_num_tokens=128)
 
     # Chunked prefill multi-request — V2 matches V1
@@ -319,7 +339,8 @@ def test_chunked_prefill_with_eviction(self):
         )
 
     # Eviction + block reuse — V2 matches V1
-    def test_eviction_with_block_reuse(self):
+    # Traced while nvbugs 6462303 ("Test terminated unexpectedly") is open.
+    def test_eviction_with_block_reuse(self, trace_executor_loop):
         _run_eviction_test(
             self.MODEL_PATH,
             EVICTION_PROMPTS_SHORT,

tests/integration/test_lists/waives.txt

@@ -357,8 +357,6 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:in
 full:RTX_PRO_6000_Blackwell_Server_Edition/perf/test_perf.py::test_perf[quant:w4a8_awq] SKIP (https://nvbugs/5161074)
 full:sm100/examples/test_nemotron.py::test_llm_nemotron_3_8b_1gpu[bfloat16-fp8] SKIP (megatron-core 0.8 is not supported in python 3.12)
 full:sm100/unittest/bindings SKIP (Disable for Blackwell)
-kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill SKIP (https://nvbugs/6428002)
-kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse SKIP (https://nvbugs/6462303)
 llmapi/test_llm_api_pytorch_bart.py::test_bart_pytorch_generate_encoder_decoder_end_to_end[bf16-kv-v1-cuda-graph-on-greedy-tp2-bart-large-cnn] SKIP (https://nvbugs/6463812)
 llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[cudagraph] SKIP (https://nvbugs/6463829)
 llmapi/test_llm_api_pytorch_moe_lora.py::test_mixtral_moe_routed_expert_fp8_multi_lora_varying_ranks[eager] SKIP (https://nvbugs/6463829)