From 419ed51a0fa6b748e5661be90d34a8f8ce846a25 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Mon, 21 Sep 2026 17:11:40 +1000 Subject: [PATCH] Add `cell_refs` to find sigil references in a notebook cell --- DEV.md | 2 ++ src/lib.rs | 2 +- src/nb.rs | 47 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/DEV.md b/DEV.md index 592c44f..d16c61b 100644 --- a/DEV.md +++ b/DEV.md @@ -52,4 +52,6 @@ Rust callers can consume a `StreamIter` with `cancel_and_join()` to cancel, drai Notebook hierarchy uses `heading_level(source)`, `section_range(levels, idx)` and `ancestor_indices(levels, idx)`. Callers supply zero for non-heading cells. Heading detection skips blank lines and lines starting with `#|`, then checks the first remaining line against `^#{1,6} \w`. It does not search past ordinary text. Section ranges include the addressed cell and end before the next equal-or-higher heading; non-headings select themselves. Ancestors exclude the addressed cell and are returned outermost first. These are calculations over the supplied levels, without retained outline state. Rustygate uses them for its cell selectors. +`cell_refs(cell)` takes an nbformat cell as a `serde_json::Value` and returns its sigil references as `CellRefs { vars, cmds, tools }`, in order of appearance, with duplicates. A prompt cell has `solveit_ai: true` in its metadata. `vars` holds each `expr` written as `` $`expr` `` in a prompt cell's source. `cmds` holds each `cmd` written as `` !`cmd` `` in a prompt cell's source. `tools` holds each name written as `` &`name` `` or `` &`[a, b]` ``. A name holds word characters and dots. `cell_refs` reads `tools` from the source of a prompt or Markdown cell. For every other cell it reads the `text/markdown` data of `display_data` and `execute_result` outputs. It never reads a prompt cell's outputs. This function is Rust-only. Rustygate uses it for the cells API's `refs=true`. + This package intentionally has no CLI. Python is the interface. diff --git a/src/lib.rs b/src/lib.rs index e141dcd..c2d2c51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ mod walk; mod python; pub use block::{BlockIter, SearchBlock, block_iter}; -pub use nb::{NbCell, NbIter, NbOptions, ancestor_indices, heading_level, nb_iter, nb_search, nb_search_file, section_range, sigil_exprs, sigil_names}; +pub use nb::{CellRefs, NbCell, NbIter, NbOptions, ancestor_indices, cell_refs, heading_level, nb_iter, nb_search, nb_search_file, section_range}; pub use search::{MatchSpan, RgIter, RgOptions, SearchKind, SearchLine, compile_regex, rg, rg_iter, search_path, search_text}; pub use walk::{FindIter, FindOptions, StreamIter, find, find_iter}; diff --git a/src/nb.rs b/src/nb.rs index d9aef54..03a1c8a 100644 --- a/src/nb.rs +++ b/src/nb.rs @@ -42,9 +42,9 @@ pub fn ancestor_indices(levels: &[usize], idx: usize) -> Vec { parents } -/// Group 1 of every `` sigil`body` `` match in `text`, in order of appearance. -fn sigil_caps(text: &str, sigil: char, body: &str) -> Vec { - let re = RegexMatcher::new(&format!(r"\x{{{:x}}}`({body})`", sigil as u32)).expect("sigil pattern compiles"); +/// Group 1 of every `` sigil`body` `` match in `text`, in order of appearance. `sigil` and `body` are regex fragments. +fn sigil_caps(text: &str, sigil: &str, body: &str) -> Vec { + let re = RegexMatcher::new(&format!("{sigil}`({body})`")).expect("sigil pattern compiles"); let mut caps = re.new_captures().expect("captures allocate"); let mut res = Vec::new(); re.captures_iter(text.as_bytes(), &mut caps, |c| { @@ -54,15 +54,44 @@ fn sigil_caps(text: &str, sigil: char, body: &str) -> Vec { res } -/// Expressions referenced as `` sigil`expr` `` in `text`, in order of appearance. -pub fn sigil_exprs(text: &str, sigil: char) -> Vec { sigil_caps(text, sigil, "[^`]+") } - -/// Names referenced as `` sigil`name` `` or `` sigil`[a, b]` `` in `text`, in order of appearance. -pub fn sigil_names(text: &str, sigil: char) -> Vec { - let groups = sigil_caps(text, sigil, r"[\w.]+|\[[\w.,\s]+\]"); +/// Names written as `` &`name` `` or `` &`[a, b]` `` in `text`. A name holds word characters and dots. +fn tool_names(text: &str) -> Vec { + let groups = sigil_caps(text, "&", r"[\w.]+|\[[\w.,\s]+\]"); groups.iter().flat_map(|g| g.split(['[', ']', ','])).map(str::trim).filter(|s| !s.is_empty()).map(String::from).collect() } +/// nbformat multiline text, a string or a list of strings, as one string. +fn nb_text(v: &serde_json::Value) -> String { + match v { serde_json::Value::String(s) => s.clone(), serde_json::Value::Array(a) => a.iter().filter_map(|o| o.as_str()).collect(), _ => String::new() } +} + +/// The sigil references in one notebook cell, in order of appearance. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct CellRefs { + /// Each `expr` written as `` $`expr` `` + pub vars: Vec, + /// Each `cmd` written as `` !`cmd` `` + pub cmds: Vec, + /// Each name written as `` &`name` `` or `` &`[a, b]` `` + pub tools: Vec, +} + +/// The sigil references in `cell`, an nbformat cell. A prompt cell has `solveit_ai: true` in its metadata. +/// `vars` and `cmds` come from a prompt cell's source. `tools` comes from the source of a prompt or Markdown cell. +/// For every other cell, `tools` comes from the `text/markdown` data of its `display_data` and `execute_result` outputs. +/// A prompt cell's outputs are never read. +pub fn cell_refs(cell: &serde_json::Value) -> CellRefs { + let src = nb_text(&cell["source"]); + let prompt = cell["metadata"]["solveit_ai"] == true; + let mut res = CellRefs::default(); + if prompt { (res.vars, res.cmds) = (sigil_caps(&src, r"\$", "[^`]+"), sigil_caps(&src, "!", "[^`]+")); } + if prompt || cell["cell_type"] == "markdown" { res.tools = tool_names(&src); return res; } + for o in cell["outputs"].as_array().into_iter().flatten() { + if matches!(o["output_type"].as_str(), Some("display_data" | "execute_result")) { res.tools.extend(tool_names(&nb_text(&o["data"]["text/markdown"]))); } + } + res +} + #[derive(Debug, Clone)] pub struct NbOptions { pub root: PathBuf,