From aada68aec158d8a091074675415f9f98b5a81508 Mon Sep 17 00:00:00 2001 From: AlexMikhalev Date: Wed, 12 Aug 2026 11:07:51 +0100 Subject: [PATCH 1/2] fix(grep): report truthful insufficient-result stats --- crates/terraphim_grep/src/lib.rs | 60 ++++++++++++++++++- .../terraphim_grep/tests/no_thesaurus_cli.rs | 18 ++++++ ...erraphim-grep-truthful-stats-2026-08-12.md | 20 +++++++ 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 docs/plans/design-terraphim-grep-truthful-stats-2026-08-12.md diff --git a/crates/terraphim_grep/src/lib.rs b/crates/terraphim_grep/src/lib.rs index 2f658f9..586716b 100644 --- a/crates/terraphim_grep/src/lib.rs +++ b/crates/terraphim_grep/src/lib.rs @@ -158,17 +158,20 @@ impl TerraphimGrep { .await } sufficiency_judge::Sufficiency::Insufficient(chunks) => { + // Preserve the retrieved chunks and KG concepts and derive the + // counters from the actual vectors so the JSON stats stay + // truthful even when the judge deems the result insufficient. let stats = GrepStats { search_latency_ms, rlm_latency_ms: None, - chunks_returned: 0, - kg_hits: 0, + chunks_returned: chunks.len(), + kg_hits: hybrid_results.kg_concepts.len(), }; Ok(GrepResult { chunks, answer: None, - concepts: vec![], + concepts: hybrid_results.kg_concepts, sufficiency: SufficiencyState::RlmInsufficient, stats, }) @@ -438,6 +441,57 @@ mod tests { assert_eq!(result.stats.kg_hits, 0); } + /// When the sufficiency judge returns `Insufficient` with non-empty chunks + /// (fewer than `min_results` matches), the returned chunks and KG concepts + /// must be preserved and the stats must be truthful: + /// `stats.chunks_returned == chunks.len()` and `stats.kg_hits == concepts.len()`. + /// This guards the JSON result invariant that blocks release wrapper #3208. + #[cfg(feature = "code-search")] + #[tokio::test] + async fn rlm_insufficient_preserves_chunks_and_reports_truthful_stats() { + let tmp = tempfile::TempDir::new().expect("tempdir"); + // Single match => chunks.len() (1) < min_results (3) => Insufficient branch. + let path = tmp.path().join("only_match.rs"); + std::fs::write(&path, "fn unique_target() { /* unique_target */ }\n").unwrap(); + + let hybrid = HybridSearcher::new("test-role".to_string(), Thesaurus::new("t".to_string())) + .expect("build hybrid searcher") + .with_search_path(tmp.path().to_path_buf()); + let grep = TerraphimGrep::new(Arc::new(hybrid), Arc::new(SufficiencyJudge::default())); + + let result = grep + .search( + "unique_target", + GrepOptions { + haystack: Haystack::Code, + max_results: 50, + ..GrepOptions::default() + }, + ) + .await + .expect("search should succeed"); + + assert!( + !result.chunks.is_empty(), + "expected at least one chunk from the known-match corpus" + ); + assert!( + matches!(result.sufficiency, SufficiencyState::RlmInsufficient), + "single match must hit the Insufficient branch, got {:?}", + result.sufficiency + ); + assert_eq!( + result.stats.chunks_returned, + result.chunks.len(), + "stats.chunks_returned must equal chunks.len() in the RlmInsufficient branch" + ); + assert_eq!( + result.stats.kg_hits, + result.concepts.len(), + "stats.kg_hits must equal concepts.len() when concepts are retained" + ); + } + /// The RLM prompt for `include_answer` must embed the `AnswerSignature` /// JSON instructions so the model knows it must return structured output. #[test] diff --git a/crates/terraphim_grep/tests/no_thesaurus_cli.rs b/crates/terraphim_grep/tests/no_thesaurus_cli.rs index 46227f9..8f3a90b 100644 --- a/crates/terraphim_grep/tests/no_thesaurus_cli.rs +++ b/crates/terraphim_grep/tests/no_thesaurus_cli.rs @@ -57,4 +57,22 @@ fn cli_runs_without_thesaurus() { Some(0), "kg_hits should be zero" ); + + // Truthful-stats invariant (blocks release wrapper #3208): the reported + // counter must always match the number of chunks actually returned, even + // when the sufficiency heuristic classifies the result as RlmInsufficient + // (fewer than min_results matches, as in this single-file corpus). + let chunks_returned = result["stats"]["chunks_returned"] + .as_u64() + .expect("chunks_returned is a number") as usize; + assert_eq!( + chunks_returned, + chunks.len(), + "stats.chunks_returned must equal chunks.len() (got {chunks_returned}, chunks = {})", + chunks.len() + ); + assert!( + chunks_returned >= 1, + "known-match corpus must report at least one returned chunk" + ); } diff --git a/docs/plans/design-terraphim-grep-truthful-stats-2026-08-12.md b/docs/plans/design-terraphim-grep-truthful-stats-2026-08-12.md new file mode 100644 index 0000000..b6ae978 --- /dev/null +++ b/docs/plans/design-terraphim-grep-truthful-stats-2026-08-12.md @@ -0,0 +1,20 @@ +# Design: truthful `terraphim_grep` insufficient-result statistics + +## Problem +Published `terraphim_grep` 1.21.1 can return non-empty `chunks` with `stats.chunks_returned = 0` when the sufficiency heuristic classifies fewer than three matches as `RlmInsufficient`. This violates the JSON result invariant and blocks release wrapper #3208. + +## Decision +In `crates/terraphim_grep/src/lib.rs`, preserve returned chunks and KG concepts in the `Sufficiency::Insufficient` branch and derive counters from the actual vectors. Do not change sufficiency thresholds or reinterpret `RlmInsufficient`. + +## Tests first +Extend the real CLI known-match regression so it asserts `stats.chunks_returned == chunks.len()`. Add/extend library coverage for `RlmInsufficient` with non-empty chunks and the same invariant. Run the targeted test before implementation and retain the expected RED. + +## Acceptance +- Known-match JSON has at least one chunk. +- `stats.chunks_returned == chunks.len()` in every result branch. +- `stats.kg_hits == concepts.len()` when concepts are retained. +- Focused crate tests, fmt and clippy pass. +- Exact Release Guardian known-match smoke passes from the branch binary. + +## Non-goals +No release version bump, publish, sufficiency threshold change, unrelated crate refactor, or dependency update in this PR. From db196555f6710e079c1e62988a400615748d5f8c Mon Sep 17 00:00:00 2001 From: AlexMikhalev Date: Wed, 12 Aug 2026 12:06:54 +0100 Subject: [PATCH 2/2] test(grep): prove insufficient KG preservation --- crates/terraphim_grep/README.md | 2 +- crates/terraphim_grep/src/lib.rs | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/terraphim_grep/README.md b/crates/terraphim_grep/README.md index 5e56cc3..149bff8 100644 --- a/crates/terraphim_grep/README.md +++ b/crates/terraphim_grep/README.md @@ -113,7 +113,7 @@ Query ├── Sufficient ──→ Return chunks (SearchOnly) ├── NeedsSynthesis ──→ RLM fallback (if LLM configured) ├── NeedsExpansion ──→ RLM fallback with additional chunks - └── Insufficient ──→ Return empty (RlmInsufficient) + └── Insufficient ──→ Preserve retrieved chunks + KG metadata (RlmInsufficient) ``` ## Key Types diff --git a/crates/terraphim_grep/src/lib.rs b/crates/terraphim_grep/src/lib.rs index 586716b..d410d79 100644 --- a/crates/terraphim_grep/src/lib.rs +++ b/crates/terraphim_grep/src/lib.rs @@ -336,7 +336,7 @@ impl TerraphimGrep { mod tests { use super::*; #[cfg(feature = "code-search")] - use terraphim_types::Thesaurus; + use terraphim_types::{NormalizedTerm, NormalizedTermValue, Thesaurus}; #[test] fn test_grep_options_default() { @@ -454,7 +454,13 @@ mod tests { let path = tmp.path().join("only_match.rs"); std::fs::write(&path, "fn unique_target() { /* unique_target */ }\n").unwrap(); - let hybrid = HybridSearcher::new("test-role".to_string(), Thesaurus::new("t".to_string())) + let mut thesaurus = Thesaurus::new("t".to_string()); + let concept_key = NormalizedTermValue::from("unique_target"); + let concept = NormalizedTerm::new(1, concept_key.clone()) + .with_display_value("unique_target".to_string()); + thesaurus.insert(concept_key, concept); + + let hybrid = HybridSearcher::new("test-role".to_string(), thesaurus) .expect("build hybrid searcher") .with_search_path(tmp.path().to_path_buf()); let grep = TerraphimGrep::new(Arc::new(hybrid), Arc::new(SufficiencyJudge::default())); @@ -490,6 +496,14 @@ mod tests { result.concepts.len(), "stats.kg_hits must equal concepts.len() when concepts are retained" ); + assert!(result.stats.kg_hits > 0, "fixture must produce a KG hit"); + assert!( + result + .concepts + .iter() + .any(|concept| concept.name == "unique_target"), + "the known KG concept must survive the RlmInsufficient branch" + ); } /// The RLM prompt for `include_answer` must embed the `AnswerSignature`