From 3e7931efca8d060f783dda83aaee329cdd5b377b Mon Sep 17 00:00:00 2001 From: Tohar Harush Date: Sun, 30 Aug 2026 15:59:19 +0300 Subject: [PATCH] perf(extract): parallelize the Python resolution tail and memoize path work (#3008) extract() is 4.4x faster cold and 5.7x faster warm on a 501-file Python-heavy corpus, and 4.0x faster at 8,000 files on a large C#/TypeScript monorepo, with byte-identical output. detect() is 1.35x faster on a 38,666-file scan emitting an identical path set. The extraction pool now runs on a forkserver context with graphify.extract preloaded, so per-worker interpreter startup is paid once per run instead of once per worker; Windows, which has no forkserver, keeps the default context and the existing BrokenProcessPool fallback. The two Python resolution hotspots -- fact collection and the identifier walk in _resolve_cross_file_imports -- were 1.47s of a 2.01s serial tail and are now per-file functions returning plain str/int payloads that a pool produces and the parent stitches back together in input order. That order is load-bearing, since the fact lists and the per-statement import order flow into emitted edge order, and the parent keeps what only it can hold: the module-stem index, the local-symbol name maps, and the edge emission. The two changes only pay off together -- under spawn the extra pools re-import the module per worker and cost more than the walks they parallelize -- so the split is gated on a forkserver being available and on at least 60 Python files, and a pool that fails for any reason discards the whole pass and the parent redoes it in-process, where the payloads are identical. Path resolution and one Python parse per file are now memoized for the life of one run, cleared per run by both extract() and detect() so a long-lived watch process cannot carry one cycle's symlink targets or source into the next. The import_from_statement and call scans are compiled tree-sitter queries with captures re-sorted to pre-order and a full-walk fallback when a cursor hits its match limit. C# type-reference arbitration indexes sourceless nodes by label once instead of scanning the graph per unresolved reference. Ignore matching compiles its per-pattern derivations once per scan instead of parsing them once per path, with literal and anchored patterns skipping the general fnmatch cascade. At 16,000 files the AST phase becomes memory-bound -- 6.9s at 8,000 files to ~140s at 16,000, in both the before and after trees -- so the extraction-side win is gone there and only the ~6x resolution-tail win survives. Sizes above 16,000 were not measured. --- graphify/cache.py | 89 ++- graphify/detect.py | 492 +++++++++++----- graphify/extract.py | 369 ++++++++++-- graphify/extractors/csharp.py | 24 +- graphify/extractors/resolution.py | 950 ++++++++++++++++++++---------- graphify/ids.py | 8 + graphify/paths.py | 95 ++- 7 files changed, 1453 insertions(+), 574 deletions(-) diff --git a/graphify/cache.py b/graphify/cache.py index 622ba6cff9..80271831c3 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -17,6 +17,7 @@ # absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths # (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import resolve_cached # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file @@ -321,7 +322,7 @@ def _stat_key_to_absolute(key: str, anchor: Path) -> str: def _stat_index_file(root: Path) -> Path: _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out return base / "cache" / "stat-index.json" @@ -453,7 +454,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # graphify-out/cache/stat-index.json inside the analyzed source tree even when # the AST cache itself is redirected to CWD (#1774 completion). _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() + resolved = resolve_cached(p) abs_key = str(resolved) # The salt is the path component that enters the digest (relative to root, or # the absolute-path fallback). The stat-index memo MUST be keyed by it too: @@ -462,7 +463,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # path served whichever was computed first — making file_hash order-dependent # and poisoning the persisted stat-index across runs (#1989). Store one digest # per salt so alternating roots don't force re-reads. - resolved_root = root.resolve() + resolved_root = resolve_cached(root) try: resolved_rel = resolved.relative_to(resolved_root) except ValueError: @@ -547,7 +548,7 @@ def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) + abs_key = str(resolve_cached(p)) st: "os.stat_result | None" = None try: st = p.stat() @@ -606,6 +607,34 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: # source_file the same way nodes/edges/hyperedges do, so it needs the same # portable-path treatment for cache entries to round-trip correctly across # machines/checkout directories. + def _relativized(source: str) -> "str | None": + """The stored form of one ``source_file`` value, or None to leave it be.""" + sp = Path(source) + if not sp.is_absolute(): + # os.path.abspath is lexical (no symlink resolution), matching + # the symbolic relativization below. + cwd_form = Path(os.path.abspath(sp)) + try: + if cwd_form == root_resolved / sp or not cwd_form.exists(): + return None # already root-relative, or a ghost path + except OSError: + return None + sp = cwd_form + try: + rel = os.path.relpath(sp, root_resolved) + except (ValueError, OSError): + return None # out-of-root (e.g. Windows cross-drive) + if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): + return None # escaped root — keep absolute + return rel.replace(os.sep, "/") + + # A payload holds one file's symbols, so nearly every item repeats the same + # source_file — this loop asked the identical question once per node: 61,964 + # relpath calls and 61,964 Path constructions across 438 payloads in a + # profiled run (#3008). ``root_resolved`` is fixed for the call and the rest + # is a pure function of the string, so one answer per distinct value serves + # the whole payload. + relativized: "dict[str, str | None]" = {} for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): for item in payload.get(bucket, []): if not isinstance(item, dict): @@ -613,24 +642,12 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: source = item.get("source_file") if not source: continue - sp = Path(source) - if not sp.is_absolute(): - # os.path.abspath is lexical (no symlink resolution), matching - # the symbolic relativization below. - cwd_form = Path(os.path.abspath(sp)) - try: - if cwd_form == root_resolved / sp or not cwd_form.exists(): - continue # already root-relative, or a ghost path - except OSError: - continue - sp = cwd_form try: - rel = os.path.relpath(sp, root_resolved) - except (ValueError, OSError): - continue # out-of-root (e.g. Windows cross-drive) - if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): - continue # escaped root — keep absolute - item["source_file"] = rel.replace(os.sep, "/") + rel = relativized[source] + except KeyError: + rel = relativized[source] = _relativized(source) + if rel is not None: + item["source_file"] = rel def _normalize_source_file_value(src: "str | Path", root_resolved: Path) -> str: @@ -805,6 +822,29 @@ def _rewrite_id_keyed_table_keys(payload: object, fn) -> None: } +def _json_deepcopy(obj): + """Deep-copy a JSON-shaped payload: dicts, lists, immutable scalars. + + ``copy.deepcopy`` on a cache payload was 868,965 calls / 0.75s of a profiled + run (#3008), almost all of it memo bookkeeping and dispatch rather than + copying — and none of that is needed here. The payload is about to be + ``json.dumps``-ed, so it can only hold JSON types, and every JSON scalar is + immutable. + + Containers of other types are returned by reference on purpose: the two + passes that mutate a copy — :func:`_relativize_source_files_in` and + :func:`_rewrite_strings` — descend into dicts and lists only, so nothing + reachable through a tuple or a set can be written to. Unlike ``deepcopy`` + this does not preserve shared identity between two references to the same + sub-object, which is unobservable in JSON output. + """ + if isinstance(obj, dict): + return {k: _json_deepcopy(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_json_deepcopy(v) for v in obj] + return obj + + def _rewrite_strings(obj: object, fn) -> None: """Apply ``fn`` to every string VALUE reachable in ``obj``, in place. @@ -944,7 +984,7 @@ def cache_dir(root: Path = Path("."), kind: str = "ast", vintage live. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out d = base / "cache" / kind if kind == "ast": d = d / f"v{_EXTRACTOR_VERSION}-s{_AST_CACHE_SCHEMA}" @@ -1105,8 +1145,7 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a # below would then mutate the caller's dict for real. on_disk = result if isinstance(result, dict): - import copy as _copy - on_disk = _copy.deepcopy(result) + on_disk = _json_deepcopy(result) _relativize_source_files_in(on_disk, root) # Then replace the absolute root inside the ids and remaining paths, so # the entry replays portably under any root (#2257). Strictly after the @@ -1213,7 +1252,7 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: one doc on a future run, never incorrect output. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out pruned = 0 for kind in ("semantic", "semantic-deep"): semantic_dir = base / "cache" / kind diff --git a/graphify/detect.py b/graphify/detect.py index 3668eb6fcf..892f83af5e 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -11,6 +11,7 @@ import unicodedata from concurrent.futures import ThreadPoolExecutor from enum import Enum +from functools import lru_cache from pathlib import Path from typing import Callable @@ -19,7 +20,7 @@ convert_google_workspace_file, google_workspace_enabled, ) -from graphify.paths import GRAPHIFY_OUT, out_path +from graphify.paths import GRAPHIFY_OUT, clear_resolve_cache, out_path, resolve_cached class FileType(str, Enum): @@ -1248,37 +1249,6 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa return patterns -# Parsed-pattern cache: raw pattern string -> (negated, directory_only, -# path_relative, stripped_pattern). Ignore patterns are re-evaluated for every -# walked entry; parsing the same strings per entry per scan was pure waste. -# Plain dict (no LRU): the universe of keys is the distinct pattern lines in -# the corpus's ignore files, which is small and bounded per scan. A long-lived -# `graphify watch` process spanning many repos could still accumulate keys over -# time, so cap it and clear wholesale on overflow (parsing is cheap, so a rare -# full re-fill costs nothing that matters). -_PARSED_PATTERN_CACHE: dict[str, tuple[bool, bool, bool, str]] = {} -_PARSED_PATTERN_CACHE_MAX = 100_000 - - -def _parse_ignore_pattern(pattern: str) -> tuple[bool, bool, bool, str]: - """Split one gitignore-style pattern into its matching flags, cached. - - Returns (negated, directory_only, path_relative, stripped). ``stripped`` - is empty for patterns that match nothing (bare "!", bare "/"). - """ - got = _PARSED_PATTERN_CACHE.get(pattern) - if got is None: - negated = pattern.startswith("!") - raw = pattern[1:] if negated else pattern - directory_only = raw.endswith("/") - path_relative = "/" in raw.rstrip("/") - got = (negated, directory_only, path_relative, raw.strip("/")) - if len(_PARSED_PATTERN_CACHE) >= _PARSED_PATTERN_CACHE_MAX: - _PARSED_PATTERN_CACHE.clear() - _PARSED_PATTERN_CACHE[pattern] = got - return got - - # PurePath.relative_to compares casefolded parts on Windows only; mirror that # exactly, but skip the normcase pass entirely where it is a no-op (POSIX). _CASEFOLD_PATHS = os.path.normcase("Aa") != "Aa" @@ -1295,6 +1265,13 @@ def _lexical_relative( of constructing a Path object (and, on 3.12+, walking ``parents`` quadratically) per pattern per scanned entry. That construction was the dominant cost of large scans (see CHANGELOG: 76k-file vault, 50+ min). + + No longer on the scan path: ``_eval`` below derives the same relative form + by slicing the target's normcased string against a precomputed anchor + prefix, which needs no ``parts`` tuple at all (#3008). Kept because it is + the reference this equivalence is pinned against + (``test_lexical_relative_matches_pathlib_relative_to``), and because it is + the only implementation that handles a part-less anchor. """ anchor_parts = anchor.parts if not anchor_parts: @@ -1321,66 +1298,234 @@ def _lexical_relative( return _nfc("/".join(tail)) -def _match_globstar_parts( +@lru_cache(maxsize=1 << 16) +def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: + """Match an anchored gitignore pattern without letting ``*`` cross ``/``. + + Memoized on the two strings (#3008). This used to build a fresh + ``lru_cache``-wrapped recursive closure on every call, so a 46,050-path scan of + a large repo paid 2.1M cache constructions whose contents were thrown away + immediately — the memo only ever helped within a single pattern/path pair. The + function is pure in its arguments, so the cache never goes stale and needs no + per-scan clear; the bound exists only so a long-lived ``watch`` process cannot + grow it without limit. + """ + return _anchored_ignore_match( + tuple(path.split("/")), tuple(pattern.split("/")) + ) + + +def _anchored_ignore_match( path_parts: tuple[str, ...], pattern_parts: tuple[str, ...], - path_idx: int, - pattern_idx: int, - memo: dict[tuple[int, int], bool], ) -> bool: - """Recursive ``**``-aware segment match, memoized via an explicit dict. - - Lifted out of ``_match_anchored_ignore_pattern`` (was a per-call - ``@lru_cache`` closure): the decorated inner closure referenced itself, so - every call leaked a reference cycle for the GC to reclaim on this hot path. - A plain dict passed in avoids both the cycle and the per-call cache setup. + """Segment-wise match for :func:`_match_anchored_ignore_pattern`. + + The pattern is a chain of segment tests where ``**`` may absorb any number of + segments, which is exactly a small non-deterministic automaton over pattern + positions, so it is walked as one: ``frontier`` holds every pattern index the + path could be sitting at, and each path segment advances the whole set at once. + + This replaced a memoized recursion over ``(path_parts, pattern_parts, + path_idx, pattern_idx)`` (#3008). The memo was worthless in practice — the key + carries the path, so consecutive paths never share an entry — and a profile of + a 46,050-path scan showed 22.5M executions of the recursive body, each one + hashing two tuples to miss its own cache. The automaton is O(segments x + pattern parts) with no hashing and no recursion, and decides the same language: + a non-final ``**`` either steps past itself or absorbs a segment (the two + recursive branches, now the epsilon closure plus the self-loop below), a final + ``**`` demands at least one segment remain, and any other part must match its + segment positionally. """ - key = (path_idx, pattern_idx) - cached = memo.get(key) - if cached is not None: - return cached + nparts = len(pattern_parts) + last = nparts - 1 + frontier = _globstar_closure({0}, pattern_parts, last) + for segment in path_parts: + advanced: set[int] = set() + for idx in frontier: + if idx > last: + continue # pattern exhausted but the path is not: a dead state + part = pattern_parts[idx] + if part == "**": + # A trailing ``**`` accepts as soon as one segment is left to eat, + # and this loop body only runs when one is. + if idx == last: + return True + advanced.add(idx) # absorb this segment, stay put + elif fnmatch.fnmatchcase(segment, part): + advanced.add(idx + 1) + if not advanced: + return False + frontier = _globstar_closure(advanced, pattern_parts, last) + return nparts in frontier - if pattern_idx == len(pattern_parts): - result = path_idx == len(path_parts) - else: - part = pattern_parts[pattern_idx] - if part == "**": - if pattern_idx == len(pattern_parts) - 1: - result = path_idx < len(path_parts) - else: - result = _match_globstar_parts( - path_parts, pattern_parts, path_idx, pattern_idx + 1, memo - ) or ( - path_idx < len(path_parts) - and _match_globstar_parts( - path_parts, pattern_parts, path_idx + 1, pattern_idx, memo - ) - ) - else: - result = ( - path_idx < len(path_parts) - and fnmatch.fnmatchcase(path_parts[path_idx], part) - and _match_globstar_parts( - path_parts, pattern_parts, path_idx + 1, pattern_idx + 1, memo - ) + +def _globstar_closure( + states: set[int], pattern_parts: tuple[str, ...], last: int +) -> set[int]: + """Add the pattern indices reachable from *states* without consuming a segment. + + Only a non-final ``**`` is skippable: ``a/**/b`` must accept ``a/b``. A final + ``**`` is not, because it requires a segment of its own. + """ + pending = [idx for idx in states if idx <= last and pattern_parts[idx] == "**"] + if not pending: + return states + reached = set(states) + while pending: + idx = pending.pop() + if idx < last and idx + 1 not in reached: + reached.add(idx + 1) + if idx + 1 <= last and pattern_parts[idx + 1] == "**": + pending.append(idx + 1) + return reached + + +_GLOB_METACHARS = ("*", "?", "[") + + +def _matches_ignore_pattern( + rel: str, + parts: tuple[str, ...], + parts_normcased: frozenset, + p: str, + p_normcased: str, + is_literal: bool, + path_relative: bool, + pattern_parts: tuple[str, ...] | None, + has_globstar: bool, + name_nfc: str, +) -> bool: + """Does the relative path *rel* match the bare gitignore pattern *p*? + + ``parts``/``parts_normcased``/``name_nfc`` describe the target and are derived + once per distinct relative path by the caller; the rest describe the pattern + and are derived once per scan by :func:`_compile_ignore_pattern`. Both used to + be recomputed on every one of the 3.8M calls a 46,050-path scan makes (#3008). + + Two fast paths carry almost all real-world patterns, and both are exact + rewrites of the general form below for every input this module produces — + with one documented exception, noted in the first bullet: + + - A **literal** non-anchored pattern (no ``*``, ``?`` or ``[``, and no ``/`` + by construction — a pattern containing ``/`` is anchored instead) can only + match a single path component. ``fnmatch`` reduces to equality for such a + pattern, ``name_nfc`` is ``parts[-1]`` for every ``(rel, name_nfc)`` pair + :func:`_eval` builds, and every multi-part join the general loop tries + contains a separator the pattern cannot hold. So the whole cascade + collapses to one set membership test. + + The exception is ``rel == "."``, where ``parts == (".",)`` and + ``name_nfc`` is the *directory's own* name rather than ``parts[-1]``: this + branch drops the general form's ``fnmatch(name_nfc, p)`` term, so + ``rel='.' pattern='build' name='build'`` matches under the general form and + not here. ``_eval`` produces ``rel == "."`` only when the target IS the + scan root and the pattern's anchor is a strict ancestor, and nothing asks + that question: the parent-exclusion walk never ``_eval``s the root itself, + ``collect_files`` and ``extract.py`` only ask about files, and + ``watch.py`` returns early for a directory event. It is recorded rather + than fixed because changing the fast path costs every real scan and buys + an answer no caller reads. + - An **anchored** pattern with no ``**`` matches strictly positionally, so the + automaton in :func:`_anchored_ignore_match` reduces to a length check plus a + per-segment ``fnmatchcase``. + """ + if path_relative: + if pattern_parts is None: # defensive: caller always supplies these + return _match_anchored_ignore_pattern(rel, p) + if not has_globstar: + return len(parts) == len(pattern_parts) and all( + fnmatch.fnmatchcase(a, b) for a, b in zip(parts, pattern_parts) ) - memo[key] = result - return result + return _anchored_ignore_match(parts, pattern_parts) + if is_literal: + return p_normcased in parts_normcased + + if fnmatch.fnmatch(rel, p): + return True + if fnmatch.fnmatch(name_nfc, p): + return True + for i, part in enumerate(parts): + if fnmatch.fnmatch(part, p): + return True + if fnmatch.fnmatch("/".join(parts[: i + 1]), p): + return True + return False -def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: - """Match an anchored gitignore pattern without letting ``*`` cross ``/``.""" - path_parts = tuple(path.split("/")) - pattern_parts = tuple(pattern.split("/")) - # Fast path: with no ``**`` the match is a straight segment-wise fnmatch of - # equal-length paths, so skip the recursive matcher and its memo entirely. - if "**" not in pattern_parts: - if len(path_parts) != len(pattern_parts): - return False - return all( - fnmatch.fnmatchcase(pp, qp) for pp, qp in zip(path_parts, pattern_parts) - ) - return _match_globstar_parts(path_parts, pattern_parts, 0, 0, {}) + +def _compile_ignore_pattern(anchor: Path, pattern: str) -> tuple | None: + """Pre-split one ``(anchor, pattern)`` pair into the form ``_eval`` consumes. + + Returns ``None`` for a pattern that is empty once its slashes are stripped — + the case the old inline loop skipped with ``continue``. + + The tuple carries the anchor as normcased strings so membership can be tested + by string prefix instead of ``Path.relative_to``. ``os.path.normcase`` is + identity on POSIX and, on Windows, lowercases and maps ``/`` to ``\\`` — both + length-preserving, so the slice offsets computed here stay valid against the + un-normcased path (and the case-insensitive comparison matches what + ``PureWindowsPath.relative_to`` already did). + """ + negated = pattern.startswith("!") + raw = pattern[1:] if negated else pattern + directory_only = raw.endswith("/") + path_relative = "/" in raw.rstrip("/") + p = raw.strip("/") + if not p: + return None + anchor_key = os.path.normcase(str(anchor)) + prefix = anchor_key if anchor_key.endswith(os.sep) else anchor_key + os.sep + pattern_parts = tuple(p.split("/")) if path_relative else None + return ( + anchor_key, + prefix, + len(prefix), + len(anchor.parts), + p, + os.path.normcase(p), + not any(c in p for c in _GLOB_METACHARS), + negated, + directory_only, + path_relative, + pattern_parts, + pattern_parts is not None and "**" in pattern_parts, + ) + + +# Compiled pattern lists, keyed by the identity of the caller's pattern list +# (#3008). The pattern list is fixed for a scan apart from nested ignore files, +# which are only ever *appended* as the walk descends, so a compiled list can be +# extended in place rather than rebuilt. Each entry holds a strong reference to +# the source list, which is what makes `id()` safe as a key: while an entry is +# cached its list cannot be collected, so its id cannot be reused. Entries are +# dropped wholesale on overflow — the cost of a miss is one recompile. +_IGNORE_COMPILED: "dict[int, list]" = {} +_IGNORE_COMPILED_MAX = 16 + + +def _compiled_ignore_patterns(patterns: list[tuple[Path, str]]) -> list[tuple]: + """Return the compiled form of *patterns*, compiling only what is new.""" + key = id(patterns) + entry = _IGNORE_COMPILED.get(key) + if entry is not None and entry[0] is patterns and entry[1] <= len(patterns): + if entry[1] < len(patterns): + for anchor, pattern in patterns[entry[1]:]: + compiled_one = _compile_ignore_pattern(anchor, pattern) + if compiled_one is not None: + entry[2].append(compiled_one) + entry[1] = len(patterns) + return entry[2] + + compiled: list[tuple] = [] + for anchor, pattern in patterns: + compiled_one = _compile_ignore_pattern(anchor, pattern) + if compiled_one is not None: + compiled.append(compiled_one) + if len(_IGNORE_COMPILED) >= _IGNORE_COMPILED_MAX: + _IGNORE_COMPILED.clear() + _IGNORE_COMPILED[key] = [patterns, len(patterns), compiled] + return compiled def _is_ignored( @@ -1402,10 +1547,28 @@ def _is_ignored( _cache: optional dict shared across calls within the same scan. Ancestor directory results are memoised so files under the same subtree don't re-evaluate the same patterns repeatedly. + + Performance (#3008): this was the single most expensive frame in the whole + pipeline — 437.4 s of a 469 s instrumented ``detect()`` profile on a + 46,050-path repo, 93% of the stage. The work was not the matching but the + ``Path`` arithmetic around it: ``target.relative_to(anchor)`` ran once per + pattern per path (5.1M calls, each one walking ``anchor.parents`` and + constructing ``PurePath`` objects to compare), and the per-pattern flags were + re-derived from the pattern string every time. Now the pattern list is + compiled once (:func:`_compiled_ignore_patterns`), anchor membership is a + string prefix test, and everything that depends only on the target — its + root-relative form, its NFC basename, its ``is_dir()`` — is computed at most + once per target instead of once per pattern. Decisions are unchanged; the + ``_cache`` contract is unchanged too, including that ``_eval`` writes each + target exactly once (#1235). """ if not patterns: return False + compiled = _compiled_ignore_patterns(patterns) + root_key = os.path.normcase(str(root)) + root_prefix = root_key if root_key.endswith(os.sep) else root_key + os.sep + root_prefix_len = len(root_prefix) root_nparts = len(root.parts) def _eval(target: Path) -> bool: @@ -1422,87 +1585,96 @@ def _eval(target: Path) -> bool: if _cache is not None and target in _cache: return _cache[target] - target_parts = target.parts - target_name_nfc: str | None = None + target_str = str(target) + target_key = os.path.normcase(target_str) + name_nfc = _nfc(target.name) + + # Root-relative form, needed by every non-anchored pattern whose anchor + # sits above the scan root. Computed once here rather than per pattern. + if target_key == root_key: + rel_root: str | None = "." + elif target_key.startswith(root_prefix): + rel_root = _nfc(target_str[root_prefix_len:].replace(os.sep, "/")) + else: + rel_root = None + + # One anchor typically owns many patterns (every rule in one .gitignore), + # so the anchor-relative form is derived per anchor, not per pattern. + rel_by_anchor: dict[str, str | None] = {} + # Split/normcase of a relative path, keyed by that path. A target has at + # most a handful of distinct relative forms (one per anchor, plus the + # root-relative one) but is tested against every pattern, so this is + # derived once per form rather than once per pattern. + parts_by_rel: dict[str, tuple[tuple[str, ...], frozenset]] = {} target_is_dir: bool | None = None - rel_root_known = False - rel_root: str | None = None - # rel string (or None = outside anchor) and part-count per distinct - # anchor; patterns overwhelmingly share a handful of anchors. - rel_by_anchor: dict[Path, tuple[str | None, int]] = {} - # split segments + accumulated "/" prefixes per distinct rel string. - segs_by_rel: dict[str, tuple[list[str], list[str]]] = {} - - def _segments(rel: str) -> tuple[list[str], list[str]]: - got = segs_by_rel.get(rel) - if got is None: - parts = rel.split("/") - prefixes: list[str] = [] - acc = "" - for part in parts: - acc = part if not acc else acc + "/" + part - prefixes.append(acc) - got = (parts, prefixes) - segs_by_rel[rel] = got - return got - - def _matches(rel: str, p: str, path_relative: bool) -> bool: - nonlocal target_name_nfc - if path_relative: - return _match_anchored_ignore_pattern(rel, p) - if fnmatch.fnmatch(rel, p): - return True - if target_name_nfc is None: - target_name_nfc = _nfc(target.name) - if fnmatch.fnmatch(target_name_nfc, p): - return True - parts, prefixes = _segments(rel) - for part, prefix in zip(parts, prefixes): - if fnmatch.fnmatch(part, p): - return True - if fnmatch.fnmatch(prefix, p): - return True - return False result = False - for anchor, pattern in patterns: - negated, directory_only, path_relative, p = _parse_ignore_pattern(pattern) - if not p: - continue - + for ( + anchor_key, + anchor_prefix, + anchor_prefix_len, + anchor_nparts, + p, + p_normcased, + is_literal, + negated, + directory_only, + path_relative, + pattern_parts, + has_globstar, + ) in compiled: # gitignore semantics: patterns from A/.gitignore apply ONLY to paths # under A. Matching non-anchored patterns against root-relative paths # let e.g. .hypothesis/.gitignore's bare "*" ignore the ENTIRE repo # (detect() returned 0 files). The anchor dir itself is exempt — an # ignore file governs its directory's contents, not the directory. - cached_rel = rel_by_anchor.get(anchor) - if cached_rel is None: - cached_rel = ( - _lexical_relative(target, target_parts, anchor), - len(anchor.parts), + if anchor_key in rel_by_anchor: + rel_anchor = rel_by_anchor[anchor_key] + else: + if target_key == anchor_key: + rel_anchor = "." + elif target_key.startswith(anchor_prefix): + rel_anchor = _nfc( + target_str[anchor_prefix_len:].replace(os.sep, "/") + ) + else: + rel_anchor = None # outside this anchor: cannot match + rel_by_anchor[anchor_key] = rel_anchor + if rel_anchor is None or rel_anchor == ".": + continue + + rel = rel_anchor + if not path_relative and root_nparts > anchor_nparts and rel_root is not None: + rel = rel_root + cached_parts = parts_by_rel.get(rel) + if cached_parts is None: + split = tuple(rel.split("/")) + cached_parts = ( + split, + frozenset(os.path.normcase(x) for x in split), ) - rel_by_anchor[anchor] = cached_rel - rel_anchor, anchor_nparts = cached_rel - if rel_anchor is None: - continue # target outside this pattern's anchor: cannot match - matched = False - if rel_anchor != ".": - rel = rel_anchor - if not path_relative and root_nparts > anchor_nparts: - if not rel_root_known: - rel_root_known = True - rel_root = _lexical_relative(target, target_parts, root) - if rel_root is not None: - rel = rel_root - matched = _matches(rel, p, path_relative=path_relative) - if matched and directory_only: - if target_is_dir is None: - target_is_dir = target.is_dir() - if not target_is_dir: - matched = False - - if matched: - result = not negated # last match wins; ! flips to un-ignore + parts_by_rel[rel] = cached_parts + parts, parts_normcased = cached_parts + if not _matches_ignore_pattern( + rel, + parts, + parts_normcased, + p, + p_normcased, + is_literal, + path_relative, + pattern_parts, + has_globstar, + name_nfc, + ): + continue + if directory_only: + if target_is_dir is None: + target_is_dir = target.is_dir() + if not target_is_dir: + continue + + result = not negated # last match wins; ! flips to un-ignore if _cache is not None: _cache[target] = result return result @@ -1656,13 +1828,19 @@ def _auto_follow_symlinks(root: Path) -> bool: def _resolves_under_root(path: Path, root: Path) -> bool: """True when ``path`` resolves to a target inside ``root``.""" try: - path.resolve().relative_to(root.resolve()) + resolve_cached(path).relative_to(resolve_cached(root)) except (OSError, RuntimeError, ValueError): return False return True def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: + # _resolves_under_root memoizes through resolve_cached, so a scan must start + # from a clean cache the same way extract() does (#3008). Without this, a + # `graphify watch` process would carry the previous cycle's symlink targets + # into this scan: extract() clears at its own start, which is too late for + # the detect that runs before it. + clear_resolve_cache() root = root.resolve() configured_out_dir = root / GRAPHIFY_OUT configured_out_names = {configured_out_dir.name} diff --git a/graphify/extract.py b/graphify/extract.py index 758f02f125..5c6a2030c6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -10,6 +10,7 @@ import textwrap from collections import Counter from dataclasses import dataclass, field +from functools import lru_cache from pathlib import Path, PurePath from typing import Any, Callable @@ -59,7 +60,7 @@ from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata -from graphify.paths import disambiguate_ambiguous_candidates +from graphify.paths import clear_resolve_cache, disambiguate_ambiguous_candidates, resolve_cached from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 @@ -69,15 +70,21 @@ _EXPORT_CONDITION_PRIORITY, _JS_INDEX_FILES, _JS_PRIMITIVE_TYPES, + _JS_CONFIG_DIR_CACHE, + _JS_MODULE_PATH_CACHE, _JS_RESOLVE_EXTS, + _SOURCE_KEY_CACHE, _TSCONFIG_ALIAS_CACHE, _TSCONFIG_BASEURL_CACHE, _VUE_SCRIPT_LANG_RE, _VUE_SCRIPT_RE, _WORKSPACE_MANIFEST_NAMES, + _WORKSPACE_ROOT_CACHE, _apply_symbol_resolution_facts, _augment_symbol_resolution_edges, _collect_js_symbol_resolution_facts, + _collect_python_file_facts, + _collect_python_reference_facts, _collect_python_symbol_resolution_facts, _contained_in_package, _decldef_class_stem, @@ -115,6 +122,7 @@ _python_call_identifier, _python_import_from_module, _python_imported_names, + _python_local_name_map, _python_top_level_function_bodies, _read_tsconfig_aliases, _resolve_c_include_path, @@ -141,6 +149,7 @@ _walk_js_tree, _walk_python_tree, _workspace_globs, + clear_python_tree_cache, ) from graphify.symbol_resolution import resolve_bash_source_edges # noqa: E402 @@ -207,7 +216,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: (ambiguous -> leave dangling, as before). Files whose package root IS the scan root are skipped (ids already coincide).""" try: - root = Path(root).resolve() + root = resolve_cached(root) except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} @@ -216,13 +225,13 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: if p.suffix.lower() not in (".py", ".pyi"): continue try: - rel = Path(p).resolve().relative_to(root) + rel = resolve_cached(p).relative_to(root) except (ValueError, OSError): continue parts = rel.parts if len(parts) < 2: continue # top-level file: scan-root-relative id already matches - d = Path(p).resolve().parent + d = resolve_cached(p).parent levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. @@ -1416,7 +1425,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: tf = e.get("target_file") if tf: try: - deferred_files.add(str(Path(tf).resolve())) + deferred_files.add(str(resolve_cached(tf))) except OSError: deferred_files.add(str(tf)) # `(? None: continue if resolved_file is not None: try: - if str(resolved_file.resolve()) in deferred_files: + if str(resolve_cached(resolved_file)) in deferred_files: continue except OSError: pass @@ -1453,7 +1462,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: # in-function dynamic import through here too, which would turn an edge case # into the common one — a hub module deferred from eight functions of the same # file would carry eight identical arrows. - emit_key = str(resolved_file.resolve()) if resolved_file is not None else raw + emit_key = str(resolve_cached(resolved_file)) if resolved_file is not None else raw if emit_key in rescued_targets: continue rescued_targets.add(emit_key) @@ -2276,11 +2285,31 @@ def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[ }) +@lru_cache(maxsize=1 << 16) +def _lower_suffix(source_file: str) -> str: + """A path's lowercased extension, without building a `Path` (#3008). + + `Path(...).suffix` parses the whole path into components — 1.7M of those + accesses in a profiled 32k-file run, 5.7s, almost all of them re-asking about + a path already seen. `splitext` answers from the string alone. Two + differences from `Path.suffix`: a trailing-dot name, where it reports `"."` + and pathlib reports `""` (normalized away below), and a trailing separator, + where `_lower_suffix("foo.go/")` is `""` while `Path("foo.go/").suffix` is + `".go"` — pathlib drops the empty final component and `splitext` does not. + The second is left as-is: both callers are fed node `source_file` values, + which never carry a trailing slash. Purely a function of the + string, so unlike the filesystem caches this one needs no per-run clear; the + bound is there only so a long-lived `watch` process cannot grow without limit. + """ + suffix = os.path.splitext(source_file)[1].lower() + return "" if suffix == "." else suffix + + def _lang_is_case_insensitive(source_file: object) -> bool: """True when the file's language resolves identifiers case-insensitively (#1581).""" if not source_file: return False - return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS + return _lower_suffix(str(source_file)) in _CASE_INSENSITIVE_EXTS # Language interop families for cross-file call resolution. A call in one language @@ -2326,7 +2355,7 @@ def _lang_family(source_file: object) -> str | None: """Interop family of the file's language, or None when unknown/not code.""" if not source_file: return None - return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) + return _LANG_FAMILY_BY_EXT.get(_lower_suffix(str(source_file))) # A language's own built-in throwable hierarchy, keyed by the interop family of @@ -2745,11 +2774,11 @@ def _merge_csharp_partial_class_nodes( for p in paths: if p.suffix.lower() in proj_exts: try: - project_dirs.add(p.resolve().parent) + project_dirs.add(resolve_cached(p).parent) except OSError: pass try: - stop = root.resolve() + stop = resolve_cached(root) except OSError: stop = root dir_assembly: dict[Path, str] = {} @@ -2789,7 +2818,7 @@ def _assembly_of_node(nid: str) -> str: if path is None: return "" try: - d = path.resolve().parent + d = resolve_cached(path).parent except OSError: return "" return _assembly_of_dir(d) @@ -3092,13 +3121,25 @@ def _key(label: str) -> str: if alias: import_alias_by_filenode.setdefault(e.get("source"), {})[e.get("target")] = _key(alias) + # Keyed only on nid, and this function never adds to node_by_id nor rewrites + # an existing node's source_file/label — so one answer per nid holds for the + # whole pass. The inner comprehension below re-asks about the same handful of + # module nodes per call site (63,220 calls in a profiled run, #3008). + _module_stem_keys: dict[str, str] = {} + def _module_stem_key(nid: str) -> str: + hit = _module_stem_keys.get(nid) + if hit is not None: + return hit n = node_by_id.get(nid) if not n: - return "" - sf = n.get("source_file") or "" - stem = Path(sf).stem if sf else "" - return _key(stem or n.get("label", "")) + key = "" + else: + sf = n.get("source_file") or "" + stem = Path(sf).stem if sf else "" + key = _key(stem or n.get("label", "")) + _module_stem_keys[nid] = key + return key existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} @@ -4539,7 +4580,7 @@ def extract_slnx(path: Path) -> dict: def _resolve(proj_path: str) -> str: proj_path = proj_path.replace("\\", "/") try: - return str((path.parent / proj_path).resolve()) + return str(resolve_cached(path.parent / proj_path)) except Exception: return proj_path @@ -4668,7 +4709,7 @@ def find_all(tag: str): continue ref_path_norm = ref_path.replace("\\", "/") try: - abs_ref = str((path.parent / ref_path_norm).resolve()) + abs_ref = str(resolve_cached(path.parent / ref_path_norm)) except Exception: abs_ref = ref_path_norm proj_nid = _make_id(abs_ref) @@ -4956,7 +4997,7 @@ def _xaml_project_root(path: Path) -> Path: return root boundary = _XAML_ACTIVE_EXTRACT_ROOT.resolve() try: - root.resolve().relative_to(boundary) + resolve_cached(root).relative_to(boundary) return root except ValueError: return boundary @@ -4965,7 +5006,7 @@ def _xaml_project_root(path: Path) -> Path: def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore root = _xaml_project_root(path) - cache_key = str(root.resolve()) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None + cache_key = str(resolve_cached(root)) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None if cache_key and cache_key in _XAML_CSHARP_CLASS_CACHE: return _XAML_CSHARP_CLASS_CACHE[cache_key] classes: dict[str, list[dict]] = {} @@ -5684,6 +5725,75 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: return idx, result +def _pool_context(): + """Multiprocessing context for the extraction pool, or None for the default. + + Prefers ``forkserver`` with this module preloaded. Under ``spawn`` — the + default on macOS and Windows — every worker is a fresh interpreter that + re-imports ``graphify.extract``, and that duplicated startup is paid once + per worker rather than once per run: measured worker CPU on a 501-file + corpus grew from 3.95s at 6 workers to 4.65s at 16, about 0.07s per extra + worker, and the pool phase got *slower* past 8 workers (1.06s at 8, 1.57s + at 16) because the added startup outran the added parallelism. A forkserver + imports this module once and forks each worker from that warm image, which + on the same corpus cut the pool phase to 0.60s at 12 workers and flattened + the worker-count curve so more workers help again; the cost grows with core + count, so the win is larger on a big machine, not smaller (#3008). + + Returns None when forkserver is unavailable (Windows, which only has spawn) + or unusable, leaving the executor's default context and the existing + BrokenProcessPool fallback in place. + """ + import multiprocessing + + if "forkserver" not in multiprocessing.get_all_start_methods(): + return None + try: + ctx = multiprocessing.get_context("forkserver") + # Import cost moves into the forkserver process, once. Workers inherit + # the loaded module by fork, so this must name the module the worker + # entrypoint lives in. + ctx.set_forkserver_preload(["graphify.extract"]) + return ctx + except (ValueError, OSError, ImportError): + return None + + +def _pool_worker_count(max_workers: "int | None", work_len: int) -> int: + """Worker count for a per-file pool over ``work_len`` items. + + Shared by every pool in this module so the env override, the CPU scaling and + the platform clamp are derived one way only. + + ``max_workers`` None means auto: honour the GRAPHIFY_MAX_WORKERS env + override, otherwise scale to the full CPU. The historical `, 8)` cap was a + safety bound for laptops in 2023 — on a 32-thread workstation it costs a 4x + slowdown (issue #792). Capping at ``work_len`` keeps small jobs from + spawning useless idle workers. + + Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation tied + to WaitForMultipleObjects). Clamping here keeps every path — auto-compute, + GRAPHIFY_MAX_WORKERS, and --max-workers — valid on >61-core boxes (issue + #1298). Never returns 0, so an empty work list is still a legal count. + """ + if max_workers is None: + env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() + env_cap = None + if env_raw: + try: + v = int(env_raw) + if v > 0: + env_cap = v + except ValueError: + pass + cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) + max_workers = min(cpu_cap, work_len) + + if sys.platform == "win32": + max_workers = min(max_workers, 61) + return max(max_workers, 1) + + def _extract_parallel( uncached_work: list[tuple[int, Path]], per_file: list[dict | None], @@ -5701,31 +5811,7 @@ def _extract_parallel( """ import concurrent.futures - if max_workers is None: - # Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the - # full CPU. The historical `, 8)` cap was a safety bound for laptops - # in 2023 — on a 32-thread workstation it costs a 4x slowdown - # (issue #792). Capping at len(uncached_work) keeps small jobs - # from spawning useless idle workers. - env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() - env_cap = None - if env_raw: - try: - v = int(env_raw) - if v > 0: - env_cap = v - except ValueError: - pass - cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) - max_workers = min(cpu_cap, len(uncached_work)) - - # Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation - # tied to WaitForMultipleObjects). Clamp here so every path — auto-compute, - # GRAPHIFY_MAX_WORKERS, and --max-workers — stays valid on >61-core boxes - # (issue #1298). Guard against 0 from an empty work list. - if sys.platform == "win32": - max_workers = min(max_workers, 61) - max_workers = max(max_workers, 1) + max_workers = _pool_worker_count(max_workers, len(uncached_work)) # A one-worker pool buys no parallelism: it still pays process spawn plus an # IPC round trip per file, and it is the one residual case where the parent's @@ -5746,7 +5832,9 @@ def _extract_parallel( failed: list[int] = [] # positions into uncached_work whose future failed _PROGRESS_INTERVAL = 100 try: - with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool: + with concurrent.futures.ProcessPoolExecutor( + max_workers=max_workers, mp_context=_pool_context() + ) as pool: futures = { pool.submit(_extract_single_file, item): pos for pos, item in enumerate(work_items) @@ -5853,6 +5941,108 @@ def _extract_sequential( print(f" AST extraction: {_done}/{_done} uncached files (100%)", flush=True) +# Python resolution passes below this many files stay in the parent. Both are a +# parse plus a tree walk per file — real work, but a pool has to be paid for. On +# a platform without forkserver (Windows only has spawn) every worker re-imports +# graphify.extract, ~0.07s each, so a small corpus would lose more to startup +# than the walks cost in the first place. Deliberately higher than +# _PARALLEL_THRESHOLD, which gates far heavier per-file extraction work. +_PY_PASS_PARALLEL_THRESHOLD = 60 + + +def _python_facts_worker(args: tuple) -> "tuple[int, dict | None]": + """Pool entrypoint for per-file Python symbol-resolution facts. + + Must be at module level (not a closure) so it can be pickled. + + Args: + args: ``(index, path_str, root_str)``; ``root`` anchors module resolution. + + Returns: + ``(index, payload)`` so results can be placed back in order. + """ + idx, path_str, root_str = args + return idx, _collect_python_file_facts(Path(path_str), Path(root_str)) + + +def _python_refs_worker(args: tuple) -> "tuple[int, dict | None]": + """Pool entrypoint for per-file Python cross-file-import reference facts. + + Args: + args: ``(index, path_str, name_to_nid)``. The name map is built in the + parent because node ids only exist there. + + Returns: + ``(index, payload)`` so results can be placed back in order. + """ + idx, path_str, name_to_nid = args + return idx, _collect_python_reference_facts(Path(path_str), name_to_nid) + + +def _map_python_pass( + worker, + work_items: list[tuple], + slots: int, + parallel: bool, + max_workers: "int | None", + label: str, +) -> "list[dict | None] | None": + """Run a per-file Python resolution pass in a worker pool. + + Both Python resolution hotspots are a parse plus a tree walk per file, over + files the extraction workers already parsed once, and only the cross-file + merge genuinely needs the global node table. Moving the per-file halves into + a pool leaves the parent doing just the merge: measured on a 501-file corpus + the two passes were 1.47s of a 2.01s serial tail (#3008). The payloads are + plain str/int data, and IPC is cheap enough not to matter — 4.56MB of + per-file results pickled in 0.017s on the same corpus. + + ``work_items`` are ``(slot_index, ...)`` tuples; each result lands at its own + slot, so the returned list is aligned with the caller's path list regardless + of completion order. Slots with no work item stay None. + + Returns None when the pass should run in the parent instead: pooling + disabled, too few files to pay for a pool, a single worker, or a pool that + broke — the same fallback contract as :func:`_extract_parallel`. + """ + if not parallel or len(work_items) < _PY_PASS_PARALLEL_THRESHOLD: + return None + workers = _pool_worker_count(max_workers, len(work_items)) + if workers == 1: + return None + # These passes only pay for themselves on a forkserver. Under spawn every + # worker of every pool re-imports graphify.extract, and these two pools are + # in addition to the extraction pool: on the 501-file corpus, moving the + # passes into spawn pools made the whole run *slower*, 3.59s -> 4.23s median, + # while the same change on a forkserver took it to 1.70s. A platform with no + # forkserver (Windows) keeps them in the parent (#3008). + ctx = _pool_context() + if ctx is None: + return None + + import concurrent.futures + + out: "list[dict | None]" = [None] * slots + try: + with concurrent.futures.ProcessPoolExecutor( + max_workers=workers, mp_context=ctx + ) as pool: + futures = [pool.submit(worker, item) for item in work_items] + for future in concurrent.futures.as_completed(futures): + idx, payload = future.result() + out[idx] = payload + except Exception as exc: + # Any failure discards the whole pass rather than half of it: the caller + # redoes it in-process, where the payloads are byte-identical, so a + # broken pool costs time and nothing else. + import logging + logging.getLogger(__name__).warning( + "parallel %s failed (%s); running in-process instead", label, exc + ) + return None + return out + + _PARALLEL_THRESHOLD = 20 @@ -5922,6 +6112,22 @@ def extract( _TSCONFIG_BASEURL_CACHE.clear() _XAML_CSHARP_CLASS_CACHE.clear() _MD_LINK_INDEX_CACHE.clear() + # Same contract for the two path caches added in #3008: which config a + # directory sees, and a source path's root-relative key, both answer + # questions about the filesystem with no mtime component. They exist because + # cross-file resolution asked them millions of times per run — `_source_key` + # alone was 36% of a profiled 32k-file run — so they must be cleared here, + # never made permanent, or a config added between two watch rebuilds (or a + # retargeted symlink) would never be observed again. + _JS_CONFIG_DIR_CACHE.clear() + _SOURCE_KEY_CACHE.clear() + _WORKSPACE_ROOT_CACHE.clear() + _JS_MODULE_PATH_CACHE.clear() + clear_resolve_cache() + # Same per-run contract: the Python parse cache treats each file's contents + # as fixed for one run, so a watch process must not carry a previous cycle's + # trees into this one. + clear_python_tree_cache() # Infer a common root for cache keys (use first diverging segment, not sum of all matches) try: @@ -6178,7 +6384,21 @@ def _describe_syntax_error(rel: str, line: "int | None", kept: int) -> str: # marker set in the per-file extractor. Populated just before the pass that uses it. callable_nids: set[str] = set() - _augment_symbol_resolution_edges(paths, all_nodes, all_edges, root) + # The per-file half of this pass — one parse plus two tree walks per Python + # file — needs no cross-file state, so hand it to a pool and let the parent + # do only the merge (#3008). + _py_fact_paths = [p for p in paths if p.suffix == ".py"] + _py_fact_payloads = _map_python_pass( + _python_facts_worker, + [(i, str(p), str(root)) for i, p in enumerate(_py_fact_paths)], + len(_py_fact_paths), + parallel, + max_workers, + "Python fact collection", + ) + _augment_symbol_resolution_edges( + paths, all_nodes, all_edges, root, py_facts_payloads=_py_fact_payloads + ) # Merge a header-declared class (and its methods) with its sibling-impl # definition into ONE node (C/C++/ObjC #1547/#1556). Runs BEFORE the id-remap @@ -6250,7 +6470,7 @@ def _portable_out_of_root_sf(p: Path) -> str: _remap_seen: set[Path] = set() for _p in paths: try: - _remap_seen.add(_p.resolve()) + _remap_seen.add(resolve_cached(_p)) except (OSError, RuntimeError): pass for _e in all_edges: @@ -6259,7 +6479,7 @@ def _portable_out_of_root_sf(p: Path) -> str: continue _raw_tp = Path(_tf) try: - _tp = _raw_tp.resolve() + _tp = resolve_cached(_raw_tp) except (OSError, RuntimeError): continue if _tp in _remap_seen: @@ -6330,7 +6550,7 @@ def _portable_out_of_root_sf(p: Path) -> str: rel = path.relative_to(root) except ValueError: try: - rel = path.resolve().relative_to(root) + rel = resolve_cached(path).relative_to(root) except ValueError: continue new_id = _file_node_id(rel) @@ -6339,14 +6559,14 @@ def _portable_out_of_root_sf(p: Path) -> str: # Also register the absolute-resolved form of the file-level id so # alias/workspace import targets (resolved via .resolve()) remap to # canonical instead of orphaning (#1529). - old_id_abs = _make_id(str(path.resolve())) + old_id_abs = _make_id(str(resolve_cached(path))) if old_id_abs != new_id: id_remap[old_id_abs] = new_id old_prefs: list[tuple[str, str]] = [] old_pref = _file_node_id(path) if old_pref != new_id: old_prefs.append((old_pref, new_id)) - old_pref_abs = _file_node_id(path.resolve()) + old_pref_abs = _file_node_id(resolve_cached(path)) if old_pref_abs != new_id and old_pref_abs != old_pref: old_prefs.append((old_pref_abs, new_id)) # Bash entrypoint node ids append "__entry" to the file-level id @@ -6365,10 +6585,10 @@ def _portable_out_of_root_sf(p: Path) -> str: if _entry_old != _entry_new: id_remap.setdefault(_entry_old, _entry_new) if old_prefs: - prefix_remap[path.resolve()] = old_prefs + prefix_remap[resolve_cached(path)] = old_prefs # Absolute form first: it is the longest, so prefix decomposition can # try forms in order without a shorter form shadowing it. - stem_forms[path.resolve()] = ( + stem_forms[resolve_cached(path)] = ( new_id, [old_pref_abs, old_pref, new_id] ) if id_remap: @@ -6420,7 +6640,7 @@ def _portable_out_of_root_sf(p: Path) -> str: if n.get("type") == "package": continue try: - entry = prefix_remap.get(Path(sf).resolve()) + entry = prefix_remap.get(resolve_cached(sf)) except Exception: continue if entry is None: @@ -6526,7 +6746,7 @@ def _edge_key(edge: dict) -> str: def _decompose(target: str, tf: str) -> "tuple[str, str] | None": try: - forms = stem_forms.get(Path(tf).resolve()) + forms = stem_forms.get(resolve_cached(tf)) except (OSError, RuntimeError): return None if not forms: @@ -6654,12 +6874,37 @@ def _learn(e: dict) -> None: py_paths = [p for p in paths if p.suffix == ".py"] if py_paths: py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] + # Local symbol maps are built here because node ids only exist in the + # parent, and only after the id-remap above; the walk that consumes them + # is per-file, so it goes to a pool (#3008). A file with no local symbols + # is dropped from the work list, matching the in-process skip. + _ref_name_maps = [ + _python_local_name_map(r, str(p)) for r, p in zip(py_results, py_paths) + ] + _ref_payloads = _map_python_pass( + _python_refs_worker, + [ + (i, str(p), nm) + for i, (p, nm) in enumerate(zip(py_paths, _ref_name_maps)) + if nm + ], + len(py_paths), + parallel, + max_workers, + "Python reference collection", + ) try: - cross_file_edges = _resolve_cross_file_imports(py_results, py_paths) + cross_file_edges = _resolve_cross_file_imports( + py_results, py_paths, ref_payloads=_ref_payloads + ) all_edges.extend(cross_file_edges) except Exception as exc: import logging logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc) + # Last consumer of the shared Python trees — release them here rather + # than at the end of the run, so the remaining passes do not hold a + # parse tree per Python file for no reason. + clear_python_tree_cache() # Cross-file Java import resolution java_paths = [p for p in paths if p.suffix == ".java"] @@ -6867,6 +7112,11 @@ def _looks_like_bash(result: object) -> bool: # of these files with no import evidence is gated below (#1659). _JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") _go_module_cache: dict[Path, str | None] = {} + # The import-path filter below runs once per candidate per Go raw_call, and + # the candidate set for a common method name is large. Without this memo the + # lookup was 48% of extract()'s wall clock on a 2,500-file Go corpus, nearly + # all of it re-deriving the same answer for the same file. + _go_import_path_cache: dict[str, str | None] = {} for rc in all_raw_calls: callee = rc.get("callee", "") if not callee: @@ -6932,7 +7182,8 @@ def _looks_like_bash(result: object) -> bool: candidates = [ candidate for candidate in candidates if _go_import_path_for_file( - nid_to_source_file.get(candidate, ""), root, _go_module_cache + nid_to_source_file.get(candidate, ""), root, + _go_module_cache, _go_import_path_cache, ) == import_path ] if not candidates: @@ -7133,7 +7384,7 @@ def _sf_entry(sf: str, sf_path: Path) -> tuple[str, str, tuple[str, ...]]: canonical_id = _file_node_id(rel) new_sf = rel.as_posix() try: - sf_resolved = sf_path.resolve() + sf_resolved = resolve_cached(sf_path) except (OSError, RuntimeError): sf_resolved = sf_path # Learn the STEM (extension-dropped) forms too: symbol producers mint diff --git a/graphify/extractors/csharp.py b/graphify/extractors/csharp.py index f823d64c30..ad47946f5c 100644 --- a/graphify/extractors/csharp.py +++ b/graphify/extractors/csharp.py @@ -373,19 +373,26 @@ def _label_for_type_ref_target(target_node: dict, source_file: str) -> str | Non return alias return stem or None + # First sourceless node per label, in `all_nodes` order — the answer + # `_dangling_stub_id` used to re-derive by scanning every node on every call + # (#3008). Nothing in this pass rewrites a node's `label` or `source_file`, so + # one pass up front is equivalent; the only new members are the stubs minted + # below, which register themselves. `setdefault` keeps the earliest match, so + # a later stub never shadows a placeholder the scan would have reached first. + placeholder_by_label: dict[object, str] = {} + for node in all_nodes: + nid = node.get("id") + if isinstance(nid, str) and _is_placeholder(node): + placeholder_by_label.setdefault(node.get("label"), nid) + def _dangling_stub_id(label: str, current_target: object) -> str: current = node_by_id.get(current_target) if _is_placeholder(current) and current.get("label") == label: return str(current_target) - for node in all_nodes: - nid = node.get("id") - if ( - isinstance(nid, str) - and node.get("label") == label - and _is_placeholder(node) - ): - return nid + hit = placeholder_by_label.get(label) + if hit is not None: + return hit stem = _make_id(label) stub_id = stem @@ -404,6 +411,7 @@ def _dangling_stub_id(label: str, current_target: object) -> str: } all_nodes.append(node) node_by_id[stub_id] = node + placeholder_by_label.setdefault(label, stub_id) return stub_id REPOINT_RELATIONS = {"implements", "inherits", "references"} diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 4982c7d05b..d6b1a430e4 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -10,6 +10,7 @@ _make_id, _read_text, ) +from graphify.paths import resolve_cached import hashlib import json import os @@ -22,8 +23,32 @@ # compilerOptions.baseUrl per config path, as an absolute dir (#2153). _TSCONFIG_BASEURL_CACHE: "dict[str, Path | None]" = {} +# Nearest tsconfig/jsconfig per starting directory, including the negative answer +# (#3008). The two loaders below cache config *contents* but each called +# _find_js_config unconditionally first, so the upward directory walk — two +# `exists()` probes per ancestor — ran on every import in the corpus. On a +# 32k-file corpus that was 2.37M of the run's 2.55M `exists()` calls. +_JS_CONFIG_DIR_CACHE: "dict[str, tuple[Path, Path] | None]" = {} + +# `_source_key` results per (source_file, root). The hottest function in the +# whole pipeline: 1.7M calls / 36% of a profiled 32k-file run, each doing an +# unmemoized `resolve()` over a handful of distinct paths (#3008). +_SOURCE_KEY_CACHE: "dict[tuple[str, str], str]" = {} + _WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") +# Nearest workspace root per starting directory, negative answer included (#3008). +# `_load_workspace_packages` already caches the package map it builds, but it called +# `_find_workspace_root` first on every one of its 13,892 calls in a profiled +# 32k-file run — and that walk reads and json-parses the `package.json` of every +# ancestor that has one, looking for a `workspaces` key. Same shape as +# `_JS_CONFIG_DIR_CACHE`, and cleared per run for the same reason. +_WORKSPACE_ROOT_CACHE: "dict[str, Path | None]" = {} + +# Resolved module specifier per `(specifier, starting directory)`, negative answer +# included (#3008). See :func:`_resolve_js_module_path`; cleared per run. +_JS_MODULE_PATH_CACHE: "dict[tuple[str, str], Path | None]" = {} + _JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs", ".cjs") _JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") @@ -132,7 +157,7 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. if not ext or ext.startswith("@"): continue - extended_path = (base_dir / ext).resolve() + extended_path = resolve_cached(base_dir / ext) if not extended_path.suffix: extended_path = extended_path.with_suffix(".json") if extended_path.exists(): @@ -194,14 +219,26 @@ def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": configures resolution in jsconfig.json got no aliases at all (#2153). tsconfig.json wins when both sit in one directory, matching tsc and editors, which consult jsconfig.json only when there is no tsconfig.json. + + Memoized per starting directory, negative answers included, and cleared per + run alongside the two content caches below (#2917, #3008) — so a config + added or removed between two extract() calls in one watch process is seen. """ - current = start_dir.resolve() + current = resolve_cached(start_dir) + key = str(current) + if key in _JS_CONFIG_DIR_CACHE: + return _JS_CONFIG_DIR_CACHE[key] + found: "tuple[Path, Path] | None" = None for candidate in [current, *current.parents]: for name in ("tsconfig.json", "jsconfig.json"): config = candidate / name if config.exists(): - return config, candidate - return None + found = config, candidate + break + if found is not None: + break + _JS_CONFIG_DIR_CACHE[key] = found + return found def _load_tsconfig_aliases(start_dir: Path) -> dict[str, list[str]]: """Walk up from start_dir to find tsconfig/jsconfig.json and return compilerOptions.paths aliases. @@ -325,10 +362,15 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], return first def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() + current = resolve_cached(start_dir) + key = str(current) + if key in _WORKSPACE_ROOT_CACHE: + return _WORKSPACE_ROOT_CACHE[key] + found: "Path | None" = None for candidate in [current, *current.parents]: if (candidate / "pnpm-workspace.yaml").exists(): - return candidate + found = candidate + break package_json = candidate / "package.json" if package_json.is_file(): try: @@ -336,8 +378,10 @@ def _find_workspace_root(start_dir: Path) -> Path | None: except Exception: continue if "workspaces" in data: - return candidate - return None + found = candidate + break + _WORKSPACE_ROOT_CACHE[key] = found + return found def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: globs: list[str] = [] @@ -434,7 +478,7 @@ def _contained_in_package(resolved: Path, package_dir: Path) -> bool: (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay within package_dir after resolution.""" try: - return resolved.resolve().is_relative_to(package_dir.resolve()) + return resolve_cached(resolved).is_relative_to(resolve_cached(package_dir)) except ValueError: return False @@ -517,6 +561,23 @@ def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> P return _resolve_js_import_path(raw) if start_dir is None: return _resolve_js_import_path(Path(raw)) + + # A specifier resolves the same way for every import that writes it from the + # same directory, and the corpus writes the same handful over and over — + # `./utils`, a package name, an alias — so this was 76,317 calls at ~174us + # each, each one probing the filesystem for candidate extensions and index + # files (#3008). Keyed by directory rather than by file, so the hit rate is + # per-directory. Filesystem-dependent, hence the per-run clear. + cache_key = (raw, str(start_dir)) + if cache_key in _JS_MODULE_PATH_CACHE: + return _JS_MODULE_PATH_CACHE[cache_key] + resolved = _resolve_js_module_path_uncached(raw, start_dir) + _JS_MODULE_PATH_CACHE[cache_key] = resolved + return resolved + + +def _resolve_js_module_path_uncached(raw: str, start_dir: Path) -> Path | None: + """The filesystem probing behind :func:`_resolve_js_module_path`.""" if raw.startswith("."): return _resolve_js_import_path(start_dir / raw) @@ -563,7 +624,7 @@ def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": """ if not raw: return None - candidate = (Path(str_path).parent / raw).resolve() + candidate = resolve_cached(Path(str_path).parent / raw) if candidate.is_file(): return candidate return None @@ -641,13 +702,26 @@ def _blank(s: str) -> str: return "".join(out), lang def _source_key(source_file: str, root: Path) -> str: + """A source path as a root-relative key, memoized for the run (#3008). + + Pure in ``(source_file, root)``, but called once per node per collision + group — 1.7M times on a 32k-file corpus, over only ~46k distinct paths — and + each call paid a full symlink walk. Cleared per run with the other + path caches (#2917). + """ if not source_file: return "" + cache_key = (source_file, str(root)) + hit = _SOURCE_KEY_CACHE.get(cache_key) + if hit is not None: + return hit source_path = Path(source_file) try: - return str(source_path.resolve().relative_to(root)) + value = str(resolve_cached(source_path).relative_to(root)) except Exception: - return str(source_path) + value = str(source_path) + _SOURCE_KEY_CACHE[cache_key] = value + return value def _node_disambiguation_source_key(node: dict, root: Path) -> str: source_file = str(node.get("source_file", "")) @@ -815,7 +889,7 @@ def _js_source_path(source_file: str, root: Path) -> Path | None: if not path.is_absolute(): path = root / path try: - return path.resolve() + return resolve_cached(path) except Exception: return path @@ -839,8 +913,8 @@ def _apply_symbol_resolution_facts( ): return - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + path_by_resolved = {resolve_cached(path): path for path in paths} + source_file_id = {resolve_cached(path): _make_id(str(path)) for path in paths} symbol_nodes: dict[tuple[Path, str], str] = {} for node in nodes: source_path = _js_source_path(str(node.get("source_file", "")), root) @@ -851,7 +925,7 @@ def _apply_symbol_resolution_facts( symbol_nodes[(source_path, label)] = str(node["id"]) def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) existing = symbol_nodes.get((resolved_path, name)) if existing is not None: return existing @@ -911,15 +985,15 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() + file_path = resolve_cached(import_fact.file_path) local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), + resolve_cached(import_fact.target_path), import_fact.imported_name, ) pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + pending_aliases_by_file.setdefault(resolve_cached(alias_fact.file_path), []).append(alias_fact) for file_path, aliases in pending_aliases_by_file.items(): local_aliases = local_aliases_by_file.setdefault(file_path, {}) @@ -938,8 +1012,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() + source_path = resolve_cached(star_fact.file_path) + target_path = resolve_cached(star_fact.target_path) star_exports_by_file.setdefault(source_path, []).append(target_path) source_id = source_file_id.get(source_path) if source_id is not None: @@ -955,8 +1029,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() + source_path = resolve_cached(namespace_fact.file_path) + target_path = resolve_cached(namespace_fact.target_path) namespace_id = ensure_symbol_node( namespace_fact.file_path, namespace_fact.exported_name, @@ -987,10 +1061,10 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() + file_path = resolve_cached(export_fact.file_path) origin: tuple[Path, str] | None = None if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) + origin = (resolve_cached(export_fact.target_path), export_fact.target_name) elif export_fact.local_name is not None: origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) if origin is None and (file_path, export_fact.local_name) in symbol_nodes: @@ -1013,7 +1087,7 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() + target_path = resolve_cached(target_path) key = (target_path, imported_name) if seen is None: seen = set() @@ -1033,7 +1107,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup return key for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) + source_id = source_file_id.get(resolve_cached(import_fact.file_path)) if source_id is None: continue origin_path, origin_symbol = resolve_exported_origin( @@ -1077,7 +1151,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup # canonicalizes — or drop it when no file node id is available. owned = {str(n.get("id")) for n in nodes} for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() + file_path = resolve_cached(use_fact.file_path) target_id = None unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) if unresolved_origin is not None: @@ -1483,20 +1557,126 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut trees: dict[Path, tuple[bytes, object]] = {} + def _collect_export_facts(node, source: bytes, path: Path) -> None: + """The export-statement half of one file's facts, for one syntax node. + + Lifted out of its own pass over every tree so it can ride along with the + declaration/import walk (#3008): the walk was 27.3M node visits in a + profiled 32k-file run, and this pass re-visited every one of them to look + at export statements alone. It appends only to `exports`, + `namespace_exports` and `star_exports`, none of which the merged-in walk + touches, so every list keeps the order it had. Each `continue` in the + original loop body became a `return` — nothing followed it. + """ + raw_module = _js_module_specifier(node, source) + export_clause = _js_export_clause(node) + # `export type { X } from ...` / `export type * from ...`: the + # statement-level `type` keyword is a bare anonymous child; the + # default binding NAMED type sits inside the clause instead (#3123). + stmt_type_only = any( + child.type == "type" and not child.is_named + for child in node.children + ) + if raw_module is not None: + target_path = _resolve_js_module_path(raw_module, path.parent) + if target_path is None: + return + target_path = resolve_cached(target_path) + namespace_name = _js_namespace_export_name(node, source) + if namespace_name is not None: + facts.namespace_exports.append( + _NamespaceExportFact( + path, + namespace_name, + target_path, + node.start_point[0] + 1, + type_only=stmt_type_only, + ) + ) + elif _js_export_statement_is_star(node): + facts.star_exports.append( + _StarExportFact(path, target_path, node.start_point[0] + 1, + type_only=stmt_type_only) + ) + if export_clause is not None: + for original_name, exported_name in _js_named_specifiers( + export_clause, source, "export_specifier" + ): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + target_path=target_path, + target_name=original_name, + type_only=stmt_type_only, + ) + ) + return + + if export_clause is not None: + for local_name, exported_name in _js_named_specifiers( + export_clause, source, "export_specifier" + ): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + local_name=local_name, + ) + ) + return + + for exported_name in _js_exported_declaration_names(node, source): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + local_name=exported_name, + ) + ) + + # `export default class Foo {}` / `export default foo` exposes the + # symbol under the name "default"; record that so a default import + # (imported_name="default") resolves to it. `export { X as default }` + # is already handled via the export_clause path above. + default_name = _js_default_export_name(node, source) + if default_name is not None: + facts.exports.append( + _SymbolExportFact( + path, + "default", + node.start_point[0] + 1, + local_name=default_name, + ) + ) + for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = _parse_js_tree(path) if parsed is None: continue source, root_node = parsed trees[resolved_path] = parsed + # Declarations, imports, lexical aliases and exports share one DFS: the walk + # was 37.2M node visits in a profiled 32k-file run and these passes ran + # back to back over the same root (#3008). Each list is appended to + # independently and DFS order is unchanged, so per-list order is identical. for node in _walk_js_tree(root_node): + for alias, target in _js_lexical_aliases(node, source): + facts.aliases.append( + _SymbolAliasFact(path, alias, target, node.start_point[0] + 1) + ) + if node.type == "export_statement": for name in _js_exported_declaration_names(node, source): facts.declarations.append( _SymbolDeclarationFact(path, name, node.start_point[0] + 1) ) + _collect_export_facts(node, source, path) if node.type != "import_statement": continue @@ -1506,7 +1686,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = resolve_cached(target_path) for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"): facts.imports.append( _SymbolImportFact( @@ -1529,110 +1709,8 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) ) - for node in _walk_js_tree(root_node): - for alias, target in _js_lexical_aliases(node, source): - facts.aliases.append( - _SymbolAliasFact(path, alias, target, node.start_point[0] + 1) - ) - for path in js_paths: - resolved_path = path.resolve() - parsed = trees.get(resolved_path) - if parsed is None: - continue - source, root_node = parsed - - for node in _walk_js_tree(root_node): - if node.type != "export_statement": - continue - - raw_module = _js_module_specifier(node, source) - export_clause = _js_export_clause(node) - # `export type { X } from ...` / `export type * from ...`: the - # statement-level `type` keyword is a bare anonymous child; the - # default binding NAMED type sits inside the clause instead (#3123). - stmt_type_only = any( - child.type == "type" and not child.is_named - for child in node.children - ) - if raw_module is not None: - target_path = _resolve_js_module_path(raw_module, path.parent) - if target_path is None: - continue - target_path = target_path.resolve() - namespace_name = _js_namespace_export_name(node, source) - if namespace_name is not None: - facts.namespace_exports.append( - _NamespaceExportFact( - path, - namespace_name, - target_path, - node.start_point[0] + 1, - type_only=stmt_type_only, - ) - ) - elif _js_export_statement_is_star(node): - facts.star_exports.append( - _StarExportFact(path, target_path, node.start_point[0] + 1, - type_only=stmt_type_only) - ) - if export_clause is not None: - for original_name, exported_name in _js_named_specifiers( - export_clause, source, "export_specifier" - ): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - target_path=target_path, - target_name=original_name, - type_only=stmt_type_only, - ) - ) - continue - - if export_clause is not None: - for local_name, exported_name in _js_named_specifiers( - export_clause, source, "export_specifier" - ): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - local_name=local_name, - ) - ) - continue - - for exported_name in _js_exported_declaration_names(node, source): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - local_name=exported_name, - ) - ) - - # `export default class Foo {}` / `export default foo` exposes the - # symbol under the name "default"; record that so a default import - # (imported_name="default") resolves to it. `export { X as default }` - # is already handled via the export_clause path above. - default_name = _js_default_export_name(node, source) - if default_name is not None: - facts.exports.append( - _SymbolExportFact( - path, - "default", - node.start_point[0] + 1, - local_name=default_name, - ) - ) - - for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1654,7 +1732,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1676,20 +1754,104 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut class_nid = _make_id(stem, class_name) _ts_walk_class_members(node, source, path, class_nid, facts) +# One Parser per process, and the parsed tree per file for the length of one +# run. Three separate phases parsed the same Python file — the worker's +# _extract_python_rationale, then _collect_python_symbol_resolution_facts and +# _resolve_cross_file_imports in the serial parent — 1,042 parses for a 350-file +# corpus, 1.32s of the 1.79s a profiled run spent inside tree_sitter.parse. The +# two parent-side phases share this cache; the worker's parse happens in another +# process and cannot (#3008). +# +# Peak memory is effectively unchanged: _collect_python_symbol_resolution_facts +# already held every Python tree at once in a local dict, and it runs after the +# node and edge lists exist, so trees and graph already coexist at the peak. +# extract() drops the cache as soon as the last consumer is done. +_PY_PARSER: "Any | None" = None +_PY_QUERIES: "dict[str, Any]" = {} +_PY_TREE_CACHE: "dict[Path, tuple[bytes, Any] | None]" = {} + +def clear_python_tree_cache() -> None: + """Drop the per-run Python parse cache. + + Must run at the start of every extract(), the same contract as + clear_resolve_cache(): the cache treats a file's contents as fixed for one + run, so a long-lived `graphify watch` process would otherwise resolve the + next cycle's symbols against the previous cycle's source text. + """ + _PY_TREE_CACHE.clear() + +def _python_language(): + import tree_sitter_python as tspython + from tree_sitter import Language + return Language(tspython.language()) + +def _python_query(pattern: str): + """Compiled tree-sitter query for ``pattern``, one per process.""" + query = _PY_QUERIES.get(pattern) + if query is None: + from tree_sitter import Query + query = Query(_python_language(), pattern) + _PY_QUERIES[pattern] = query + return query + +def _query_python_nodes(pattern: str, node, capture: str = "m") -> "list | None": + """Nodes under ``node`` matching ``pattern``, in pre-order. + + Stands in for a full Python-side walk plus a node-type test. Doing the + filtering in tree-sitter's C layer measured 3.6x faster for + `import_from_statement` and 3.1x for `call` over a 350-file corpus, where one + full walk visits 1.44M nodes to reach a few thousand of interest (#3008). + + Captures arrive in match-completion order, NOT pre-order — a nested match is + reported after every shallower one, so `a(b(c()))` beside `d()` comes back as + a, d, b, c. Sorting by start byte ascending then end byte descending + reproduces pre-order exactly, because a parent always starts no later than + its child and ends no earlier. That order is observable: callers append to + fact lists that flow into emitted edge order. + + Returns None when the cursor hit its match limit, so a caller can fall back + to walking rather than act on a silently truncated capture set. + """ + from tree_sitter import QueryCursor + cursor = QueryCursor(_python_query(pattern)) + captures = cursor.captures(node) + if cursor.did_exceed_match_limit: + return None + found = captures.get(capture) + if not found: + return [] + found.sort(key=lambda n: (n.start_byte, -n.end_byte)) + return found + def _parse_python_tree(path: Path): + key = resolve_cached(path) + if key in _PY_TREE_CACHE: + return _PY_TREE_CACHE[key] + global _PY_PARSER try: - import tree_sitter_python as tspython - from tree_sitter import Language, Parser + if _PY_PARSER is None: + from tree_sitter import Parser + _PY_PARSER = Parser(_python_language()) source = path.read_bytes() - parser = Parser(Language(tspython.language())) - return source, parser.parse(source).root_node + parsed = (source, _PY_PARSER.parse(source).root_node) except Exception: - return None + parsed = None + _PY_TREE_CACHE[key] = parsed + return parsed def _walk_python_tree(node): - yield node - for child in node.children: - yield from _walk_python_tree(child) + # Iterative DFS avoids Python's O(depth) generator-chain overhead, the same + # way _walk_js_tree does. Recursive yield-from relays every yielded value up + # through one generator frame per level, and a profiled run put 1.45M nodes + # at average depth ~17 — 24.7M frame resumptions for 1.45M nodes. + # reversed() is load-bearing: the stack pops LIFO, so children must be + # pushed right-to-left to come out left-to-right. Sibling order is + # observable — callers append to fact lists that flow into edge order. + stack = [node] + while stack: + n = stack.pop() + yield n + stack.extend(reversed(n.children)) def _python_import_from_module(node, source: bytes) -> tuple[int, str] | None: level = 0 @@ -1809,96 +1971,290 @@ def _python_call_identifier(node, source: bytes) -> str | None: return _read_text(function_node, source) return None +def _collect_python_file_facts(path: Path, root: Path) -> "dict | None": + """Symbol-resolution facts for ONE Python file, as plain picklable data. + + Split out of :func:`_collect_python_symbol_resolution_facts` so the parse and + the two tree walks can run in a worker pool: every fact here comes from this + file plus the filesystem, never from the cross-file node table, so a worker + can produce it and the parent only stitches the results back together in + input order. The parent's half of this pass was 0.85s of a 2.01s serial tail + on a 501-file corpus, ~80% of it in here (#3008). + + Values are str/int rather than Path/dataclass to keep the pickled payload + small; :func:`_apply_python_file_facts` rebuilds the fact objects. + + Returns None when the file does not parse, which the caller skips. + """ + parsed = _parse_python_tree(path) + if parsed is None: + return None + source, root_node = parsed + + imports: list[tuple[str, str, str, int]] = [] + module_imports: list[tuple[str, int, str]] = [] + exports: list[tuple[str, int, str, str]] = [] + uses: list[tuple[str, str, int]] = [] + is_init = path.name == "__init__.py" + + # An import_from_statement is a handful of nodes in a file of thousands, + # so let tree-sitter find them instead of walking every node here. + import_nodes = _query_python_nodes("(import_from_statement) @m", root_node) + if import_nodes is None: # match limit hit — fall back to the full walk + import_nodes = [n for n in _walk_python_tree(root_node) + if n.type == "import_from_statement"] + for node in import_nodes: + module = _python_import_from_module(node, source) + if module is None: + continue + level, module_name = module + target_path = _resolve_python_module_path(module_name, path, root, level) + if target_path is None: + continue + # #1146: `from pkg import submod` — if the target is a package + # (__init__.py) and an imported name matches a submodule file on + # disk, emit a file-level import edge to that submodule rather + # than only to the package. + pkg_dir = target_path.parent if target_path.name == "__init__.py" else None + target_str = str(target_path) + for imported_name, local_name in _python_imported_names(node, source): + line = node.start_point[0] + 1 + if pkg_dir is not None: + sub_py = pkg_dir / f"{imported_name}.py" + sub_pkg = pkg_dir / imported_name / "__init__.py" + submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) + if submodule is not None: + module_imports.append((str(submodule), line, local_name)) + continue + imports.append((local_name, target_str, imported_name, line)) + if is_init: + exports.append((local_name, line, target_str, imported_name)) + + for source_id, body in _python_top_level_function_bodies(path, root_node, source): + # Only `call` nodes matter here — 48k of the 1.44M nodes a full walk + # of this corpus visits. _python_call_identifier still returns None + # for a call whose function field is not a bare identifier, so the + # None check below stays. + call_nodes = _query_python_nodes("(call) @m", body) + if call_nodes is None: # match limit hit — fall back to the full walk + call_nodes = [n for n in _walk_python_tree(body) if n.type == "call"] + for node in call_nodes: + imported_name = _python_call_identifier(node, source) + if imported_name is None: + continue + uses.append((source_id, imported_name, node.start_point[0] + 1)) + + return { + "imports": imports, + "module_imports": module_imports, + "exports": exports, + "uses": uses, + } + +def _apply_python_file_facts( + py_paths: list[Path], + payloads: "list[dict | None]", + facts: _SymbolResolutionFacts, +) -> None: + """Stitch per-file Python facts into ``facts``, in input order. + + ``payloads`` must be aligned with ``py_paths`` — position i holds the facts + for py_paths[i], or None for a file that did not parse. The four fact lists + are disjoint, so appending all four per file reproduces exactly what the + sequential collector produced (each list in py_paths order); downstream edge + emission follows those lists, so the alignment is load-bearing. + """ + for path, payload in zip(py_paths, payloads): + if payload is None: + continue + for local_name, target_str, imported_name, line in payload["imports"]: + facts.imports.append( + _SymbolImportFact(path, local_name, Path(target_str), imported_name, line) + ) + for submodule_str, line, local_name in payload["module_imports"]: + facts.module_imports.append((path, Path(submodule_str), line, local_name)) + for local_name, line, target_str, target_name in payload["exports"]: + facts.exports.append( + _SymbolExportFact( + path, + local_name, + line, + target_path=Path(target_str), + target_name=target_name, + ) + ) + for source_id, imported_name, line in payload["uses"]: + facts.uses.append( + _SymbolUseFact(path, source_id, imported_name, "calls", "call", line) + ) + def _collect_python_symbol_resolution_facts( paths: list[Path], root: Path, facts: _SymbolResolutionFacts, + payloads: "list[dict | None] | None" = None, ) -> None: + """Collect Python symbol-resolution facts for every ``.py`` path. + + ``payloads`` lets a caller supply per-file facts computed elsewhere — the + extraction pool, see ``_map_python_pass`` in extract.py — one entry per + ``.py`` path in ``paths`` order. Passing None collects them in-process. + """ py_paths = [path for path in paths if path.suffix == ".py"] if not py_paths: return - - trees: dict[Path, tuple[bytes, object]] = {} - for path in py_paths: - parsed = _parse_python_tree(path) - if parsed is None: - continue - source, root_node = parsed - trees[path.resolve()] = parsed - - for node in _walk_python_tree(root_node): - if node.type != "import_from_statement": - continue - module = _python_import_from_module(node, source) - if module is None: - continue - level, module_name = module - target_path = _resolve_python_module_path(module_name, path, root, level) - if target_path is None: - continue - # #1146: `from pkg import submod` — if the target is a package - # (__init__.py) and an imported name matches a submodule file on - # disk, emit a file-level import edge to that submodule rather - # than only to the package. - pkg_dir = target_path.parent if target_path.name == "__init__.py" else None - for imported_name, local_name in _python_imported_names(node, source): - line = node.start_point[0] + 1 - if pkg_dir is not None: - sub_py = pkg_dir / f"{imported_name}.py" - sub_pkg = pkg_dir / imported_name / "__init__.py" - submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) - if submodule is not None: - facts.module_imports.append((path, submodule, line, local_name)) - continue - facts.imports.append( - _SymbolImportFact(path, local_name, target_path, imported_name, line) - ) - if path.name == "__init__.py": - facts.exports.append( - _SymbolExportFact( - path, - local_name, - line, - target_path=target_path, - target_name=imported_name, - ) - ) - - for path in py_paths: - parsed = trees.get(path.resolve()) - if parsed is None: - continue - source, root_node = parsed - for source_id, body in _python_top_level_function_bodies(path, root_node, source): - for node in _walk_python_tree(body): - imported_name = _python_call_identifier(node, source) - if imported_name is None: - continue - facts.uses.append( - _SymbolUseFact( - path, - source_id, - imported_name, - "calls", - "call", - node.start_point[0] + 1, - ) - ) + if payloads is None: + payloads = [_collect_python_file_facts(path, root) for path in py_paths] + _apply_python_file_facts(py_paths, payloads, facts) def _augment_symbol_resolution_edges( paths: list[Path], nodes: list[dict], edges: list[dict], root: Path, + py_facts_payloads: "list[dict | None] | None" = None, ) -> None: facts = _SymbolResolutionFacts() _collect_js_symbol_resolution_facts(paths, facts) - _collect_python_symbol_resolution_facts(paths, root, facts) + _collect_python_symbol_resolution_facts(paths, root, facts, payloads=py_facts_payloads) _apply_symbol_resolution_facts(paths, nodes, edges, root, facts) +def _python_local_name_map(file_result: dict, str_path: str) -> dict[str, str]: + """Local symbol name -> node id for one file's own nodes. + + Function labels end in "()"; the file node ends in ".py"; rationale nodes + never import (#563). First writer wins on a name collision (inherently + ambiguous within a file). Split out of :func:`_resolve_cross_file_imports` so + the map can be built in the parent — node ids only exist there — and handed + to a worker that walks the file (#3008). + """ + name_to_nid: dict[str, str] = {} + for n in file_result.get("nodes", []): + if n.get("source_file") != str_path or n.get("file_type") == "rationale": + continue + label = n.get("label", "") + if not label or label.endswith(".py"): + continue + sym_name = label[:-2] if label.endswith("()") else label + if sym_name and sym_name not in name_to_nid: + name_to_nid[sym_name] = n["id"] + return name_to_nid + +def _collect_python_reference_facts( + path: Path, name_to_nid: dict[str, str] +) -> "dict | None": + """Per-file half of cross-file import resolution, as plain picklable data. + + Walks the file once and returns (a) every ``from X import ...`` statement in + walk order, reduced to what the parent needs for the global stem lookup, and + (b) the first line at which each local symbol references each name. Neither + depends on the cross-file node table, so this half runs in a worker pool + while the parent keeps the global index and the edge emission. The walk was + ~87% of a 0.62s serial pass on a 501-file corpus (#3008). + + Each ``modules`` entry is ``(rel_fq, bares, names)``: + + * ``rel_fq`` is the directory-qualified stem of a relative import's target, + resolvable from this file's directory alone, or None. + * ``bares`` is every ``dotted_name`` child in child order, stopping at a + ``relative_import``. The parent maps them through its bare-stem index and + takes the first that resolves — the sequential code tried each in turn + while its target was still unresolved, and the imported names sit in that + same child list, so they were candidates too. + * ``names`` is ``(imported_name, local_name)`` per imported name, in order; + ``local_name`` honours ``import X as Y`` so a reference to the alias in the + body still attributes correctly. + + Returns None when the file does not parse. + """ + parsed = _parse_python_tree(path) + if parsed is None: + return None + source, tree_root = parsed + + modules: list[tuple[str | None, list[str], list[tuple[str, str]]]] = [] + # referenced name -> {source symbol nid: first reference line} + ref_sources: dict[str, dict[str, int]] = {} + parent_dir = path.parent + + def _text(n) -> str: + return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + + def record_import(node) -> None: + # Find the module name - handles both absolute and relative imports. + # Relative: `from .models import X` → relative_import → dotted_name + # Absolute: `from models import X` → module_name field + rel_fq: str | None = None + bares: list[str] = [] + for child in node.children: + if child.type == "relative_import": + for sub in child.children: + if sub.type == "dotted_name": + bare = _text(sub).split(".")[-1] + rel_fq = _file_stem(parent_dir / f"{bare}.py") + break + break + if child.type == "dotted_name": + bares.append(_text(child).split(".")[-1]) + + # Imported names come AFTER the 'import' keyword token. For + # `import X as Y` the target is found via X but the body uses Y. + names: list[tuple[str, str]] = [] + past_import_kw = False + for child in node.children: + if child.type == "import": + past_import_kw = True + continue + if not past_import_kw: + continue + imported_name: str | None = None + local_name: str | None = None + if child.type == "dotted_name": + imported_name = local_name = _text(child) + elif child.type == "aliased_import": + name_node = child.child_by_field_name("name") + alias_node = child.child_by_field_name("alias") + if name_node is not None: + imported_name = _text(name_node) + local_name = _text(alias_node) if alias_node is not None else imported_name + if not imported_name or not local_name: + continue + names.append((imported_name, local_name)) + modules.append((rel_fq, bares, names)) + + def visit(node, current_nid: str | None) -> None: + # node.type is an attribute read that builds a str each time, and + # this walk covers 1.4M nodes per run — read it once (#3008). + ntype = node.type + # Identifiers inside an import statement are the import itself, not a + # real use — record the import here and don't descend into it. + if ntype == "import_from_statement": + record_import(node) + return + # Attribute references to the top-level symbol that contains them: a + # class is a unit (a reference inside one of its methods counts for + # the class, matching the documented DigestAuth->Response edge), and + # a module-level function is its own source. Only set at module scope + # (current_nid is None) so nested defs never override the container. + if current_nid is None and ntype in ("class_definition", "function_definition"): + name_node = node.child_by_field_name("name") + if name_node is not None: + mapped = name_to_nid.get(_text(name_node)) + if mapped is not None: + current_nid = mapped + if ntype == "identifier" and current_nid is not None: + slot = ref_sources.setdefault(_text(node), {}) + slot.setdefault(current_nid, node.start_point[0] + 1) + for child in node.children: + visit(child, current_nid) + + visit(tree_root, None) + return {"modules": modules, "refs": ref_sources} + def _resolve_cross_file_imports( per_file: list[dict], paths: list[Path], + ref_payloads: "list[dict | None] | None" = None, ) -> list[dict]: """ Two-pass import resolution: turn file-level imports into class-level edges. @@ -1915,14 +2271,11 @@ def _resolve_cross_file_imports( BasicAuth --uses--> Request [INFERRED] """ try: - import tree_sitter_python as tspython - from tree_sitter import Language, Parser + import tree_sitter_python # noqa: F401 + import tree_sitter # noqa: F401 except ImportError: return [] - language = Language(tspython.language()) - parser = Parser(language) - # Pass 1: _file_stem(path) → {ClassName: node_id} # Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions # when multiple files share the same filename in different directories. @@ -1959,119 +2312,52 @@ def _resolve_cross_file_imports( # actually references the imported name — not to every class that merely # shares the file (#2652). The edge is anchored at the real reference, not # the import line, so `source_location` points at genuine corroboration. + # The per-file walk lives in _collect_python_reference_facts; ``ref_payloads`` + # supplies its results when a caller ran it in a worker pool, one entry per + # path in ``paths`` order (None for a file with no local symbols, no parse, + # or no payload). Passing None walks in-process. new_edges: list[dict] = [] - for file_result, path in zip(per_file, paths): + for i, (file_result, path) in enumerate(zip(per_file, paths)): str_path = str(path) - # Map each local symbol (class or function) to its node id, keyed by the - # bare symbol name. Function labels end in "()"; the file node ends in - # ".py"; rationale nodes never import (#563). First writer wins on a - # name collision (inherently ambiguous within a file). - name_to_nid: dict[str, str] = {} - for n in file_result.get("nodes", []): - if n.get("source_file") != str_path or n.get("file_type") == "rationale": - continue - label = n.get("label", "") - if not label or label.endswith(".py"): + if ref_payloads is not None: + payload = ref_payloads[i] + else: + name_to_nid = _python_local_name_map(file_result, str_path) + if not name_to_nid: continue - sym_name = label[:-2] if label.endswith("()") else label - if sym_name and sym_name not in name_to_nid: - name_to_nid[sym_name] = n["id"] - if not name_to_nid: - continue - - # Parse imports from this file - try: - source = path.read_bytes() - tree = parser.parse(source) - except Exception: + # _collect_python_file_facts already parsed this file earlier in the + # same run when both passes run in one process, so this is a cache hit. + payload = _collect_python_reference_facts(path, name_to_nid) + if payload is None: continue - # local_name -> target node id (local_name honours `import X as Y`, so a - # reference to the alias in the body still attributes correctly). + # local_name -> target node id. Insertion order is import-statement walk + # order, which is the order the edges below are emitted in. import_targets: dict[str, str] = {} - # referenced name -> {source symbol nid: first reference line} - ref_sources: dict[str, dict[str, int]] = {} - - def _text(n) -> str: - return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") - - def resolve_import(node) -> None: - # Find the module name - handles both absolute and relative imports. - # Relative: `from .models import X` → relative_import → dotted_name - # Absolute: `from models import X` → module_name field + for rel_fq, bares, names in payload["modules"]: # target_fq is the directory-qualified stem used as the key in # stem_to_entities. Relative imports are resolved exactly via the # importing file's directory; absolute imports fall back to the # bare-stem secondary index (first-writer-wins when names collide). - target_fq: str | None = None - for child in node.children: - if child.type == "relative_import": - for sub in child.children: - if sub.type == "dotted_name": - bare = _text(sub).split(".")[-1] - candidate = path.parent / f"{bare}.py" - target_fq = _file_stem(candidate) - break - break - if child.type == "dotted_name" and target_fq is None: - bare = _text(child).split(".")[-1] + target_fq = rel_fq + if target_fq is None: + for bare in bares: target_fq = bare_to_qualified.get(bare) - - if not target_fq or target_fq not in stem_to_entities: - return - - # Imported names come AFTER the 'import' keyword token. For - # `import X as Y` the target is found via X but the body uses Y. - past_import_kw = False - for child in node.children: - if child.type == "import": - past_import_kw = True - continue - if not past_import_kw: - continue - imported_name: str | None = None - local_name: str | None = None - if child.type == "dotted_name": - imported_name = local_name = _text(child) - elif child.type == "aliased_import": - name_node = child.child_by_field_name("name") - alias_node = child.child_by_field_name("alias") - if name_node is not None: - imported_name = _text(name_node) - local_name = _text(alias_node) if alias_node is not None else imported_name - if not imported_name or not local_name: - continue - tgt_nid = stem_to_entities[target_fq].get(imported_name) + if target_fq is not None: + break + if not target_fq: + continue + entities = stem_to_entities.get(target_fq) + if entities is None: + continue + for imported_name, local_name in names: + tgt_nid = entities.get(imported_name) if tgt_nid: import_targets[local_name] = tgt_nid - def visit(node, current_nid: str | None) -> None: - # Identifiers inside an import statement are the import itself, not a - # real use — resolve the import here and don't descend into it. - if node.type == "import_from_statement": - resolve_import(node) - return - # Attribute references to the top-level symbol that contains them: a - # class is a unit (a reference inside one of its methods counts for - # the class, matching the documented DigestAuth->Response edge), and - # a module-level function is its own source. Only set at module scope - # (current_nid is None) so nested defs never override the container. - if current_nid is None and node.type in ("class_definition", "function_definition"): - name_node = node.child_by_field_name("name") - if name_node is not None: - mapped = name_to_nid.get(_text(name_node)) - if mapped is not None: - current_nid = mapped - if node.type == "identifier" and current_nid is not None: - slot = ref_sources.setdefault(_text(node), {}) - slot.setdefault(current_nid, node.start_point[0] + 1) - for child in node.children: - visit(child, current_nid) - - visit(tree.root_node, None) - + ref_sources = payload["refs"] for name, tgt_nid in import_targets.items(): for src_nid, line in ref_sources.get(name, {}).items(): if src_nid == tgt_nid: @@ -2408,14 +2694,44 @@ def _go_import_path_for_file( source_file: str | Path, root: Path, module_cache: dict[Path, str | None] | None = None, + result_cache: dict[str, str | None] | None = None, +) -> str | None: + """Return the canonical Go import path for a source file inside a module. + + ``module_cache`` memoizes the directory-to-module-path lookup, so a go.mod is + read and parsed once per module rather than once per file. ``result_cache`` + memoizes the whole answer per ``source_file``, which is a different and much + larger saving: the callers ask about the same file once per candidate per raw + call, and everything before the go.mod lookup — building a ``Path``, + resolving it, and materializing ``directory.parents`` — runs on every ask. + + On a 2,500-file Go corpus this function was called 125k times and accounted + for 13.4 s of a 27.7 s ``extract()``, essentially all of it ``pathlib`` + object churn rather than filesystem work. Both caches are caller-owned and + live for one ``extract()``, so a file whose module changes between runs is + not stale. + """ + key = str(source_file) + if result_cache is not None and key in result_cache: + return result_cache[key] + + answer = _go_import_path_uncached(source_file, root, module_cache) + if result_cache is not None: + result_cache[key] = answer + return answer + + +def _go_import_path_uncached( + source_file: str | Path, + root: Path, + module_cache: dict[Path, str | None] | None = None, ) -> str | None: - """Return the canonical Go import path for a source file inside a module.""" cache = module_cache if module_cache is not None else {} path = Path(source_file) if not path.is_absolute(): path = root / path try: - directory = path.resolve().parent + directory = resolve_cached(path).parent except OSError: directory = path.absolute().parent @@ -2480,6 +2796,8 @@ def _resolve_go_type_references( contained = {edge.get("target") for edge in definition_edges if edge.get("relation") == "contains"} module_cache: dict[Path, str | None] = {} + # Many nodes share one file, so the per-file answer is asked for repeatedly. + import_path_cache: dict[str, str | None] = {} fqn_to_ids: dict[str, list[str]] = {} for node in definition_nodes: source_file = str(node.get("source_file") or "") @@ -2489,7 +2807,9 @@ def _resolve_go_type_references( or not _is_type_like_definition(node)): continue actual_path = actual_path_by_file.get(source_file, Path(source_file)) - package_path = _go_import_path_for_file(actual_path, root, module_cache) + package_path = _go_import_path_for_file( + actual_path, root, module_cache, import_path_cache + ) if package_path: fqn_to_ids.setdefault(f"{package_path}.{label}", []).append(nid) diff --git a/graphify/ids.py b/graphify/ids.py index 0143ee2d1a..9f49e76dd6 100644 --- a/graphify/ids.py +++ b/graphify/ids.py @@ -43,13 +43,21 @@ import re import unicodedata +from functools import lru_cache __all__ = ["normalize_id", "make_id"] +@lru_cache(maxsize=1 << 17) def normalize_id(s: str) -> str: r"""Normalize a single ID string to its canonical form. + Memoized (#3008): the recipe below is a pure function of the string — a + casefold/NFKC fixpoint loop plus two regex passes, ~7.6us a call — and a + corpus asks about the same names repeatedly, 448,282 calls on a 32k-file run. + The bound is a ceiling for a long-lived `watch` process, not a correctness + concern; nothing here consults the filesystem, so there is no staleness. + Guarantees, all enforced by tests: - Idempotent: ``normalize_id(normalize_id(s)) == normalize_id(s)``. diff --git a/graphify/paths.py b/graphify/paths.py index ba15b32cc7..698962acaa 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -16,6 +16,7 @@ from __future__ import annotations +import functools import json import os import re @@ -25,6 +26,47 @@ GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +# Memoized `Path.resolve()` results, keyed by the input path string (a relative +# input is keyed under its cwd-joined form, so a chdir cannot serve a wrong +# answer). See :func:`resolve_cached`. +_RESOLVE_CACHE: "dict[str, Path]" = {} + + +def resolve_cached(path: "str | Path") -> Path: + """``Path(path).resolve()``, memoized for the life of one extraction run. + + ``Path.resolve()`` is not a string operation: it walks the path component by + component through ``lstat`` to follow symlinks, so one call on a path 15 + segments deep costs ~15 syscalls. Cross-file resolution asks the same + question about the same few thousand paths millions of times — a profile of + a 32k-file corpus showed 3.9M ``realpath`` calls issuing 59M ``lstat`` + syscalls, with ``_source_key`` alone accounting for 1.7M of them (#3008). + + The filesystem is treated as frozen for the duration of a run, which is the + same assumption the tsconfig/workspace caches already make. Like those, this + cache has no mtime or content component, so ``extract()`` clears it per run + (#2917) — a symlink retargeted between two ``extract()`` calls in one + ``graphify watch`` process is observed on the next run, not within a run. + + The returned ``Path`` is shared between callers. ``Path`` is immutable, so + that is safe, but callers must not rely on object identity meaning anything. + """ + key = str(path) + if not os.path.isabs(key): + # A relative input resolves against the cwd, and watch.py can chdir to + # recover a lost repo root — so the cwd has to be part of the identity. + key = os.path.join(os.getcwd(), key) + hit = _RESOLVE_CACHE.get(key) + if hit is None: + hit = path.resolve() if isinstance(path, Path) else Path(path).resolve() + _RESOLVE_CACHE[key] = hit + return hit + + +def clear_resolve_cache() -> None: + """Drop the memoized ``resolve()`` results. Called once per ``extract()`` run.""" + _RESOLVE_CACHE.clear() + def _atomic_replace(path: "str | Path", write_fn) -> None: """Atomically replace ``path`` with content written by ``write_fn(f)``. @@ -130,9 +172,19 @@ def write_json_atomic(path: "str | Path", obj, *, indent: "int | None" = None, e ) +@functools.lru_cache(maxsize=65536) def _is_test_path(path: str) -> bool: """Classify a source path as a test path (case-insensitive, segment-aware). + Cached because it is a pure function of the string — no filesystem access — + and `disambiguate_ambiguous_candidates` asks it once per candidate per + ambiguous call site. On a 2,500-file Go corpus that was 173k calls over a few + thousand distinct paths, ~2.2 s of a run, nearly all of it re-deriving the + same answer via a fresh `PurePosixPath` and up to nine regex matches. + + The bound matters: `watch` keeps one process alive across many extractions, + so an unbounded cache would grow with every path ever classified. + Shared by extract.py and symbol_resolution.py so cross-file call resolution treats test mocks/stubs identically. A path is a test path when: * any whole path segment equals a known test dir name @@ -166,6 +218,25 @@ def _is_test_path(path: str) -> bool: return False +@functools.lru_cache(maxsize=16384) +def _posix_parent_parts(norm: str) -> tuple[str, ...]: + """``PurePosixPath(norm).parent.parts``, memoized on the normalized string. + + ``_path_proximity_winner`` is called once per ambiguous bare-name call and + walks every candidate, so a corpus with a few thousand distinct source files + built tens of thousands of ``PurePosixPath`` objects to read one attribute — + pathlib's parser accounted for ~15% of extraction CPU in a profiled run + (#3008). The parse is a pure function of the string, so one construction per + distinct path serves every call. + + Deliberately still pathlib rather than ``str.rsplit``: ``.parent`` collapses + duplicate slashes, drops ``.`` components, yields ``"."`` for a bare + filename and ``"/"`` for a root child. Those cases feed the god-node guard, + where a wrong parent silently changes which candidate wins. + """ + return PurePosixPath(norm).parent.parts + + def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) -> str | None: """Pick the candidate whose source file is closest to the call site. @@ -183,19 +254,24 @@ def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) if not call_site_file: return None call_norm = str(call_site_file).replace("\\", "/") - call_dir = PurePosixPath(call_norm).parent + # All three tiers below need each candidate's separator-normalized path, and + # two of them need its parent. Normalize once per candidate instead of once + # per candidate per tier; dict order is preserved, so tier 3's scoring order + # is unchanged. + norm_items = [(cid, str(f).replace("\\", "/")) for cid, f in candidate_files.items()] # Tier 1: exact same file. - same_file = [cid for cid, f in candidate_files.items() - if str(f).replace("\\", "/") == call_norm] + same_file = [cid for cid, norm in norm_items if norm == call_norm] if len(same_file) == 1: return same_file[0] if len(same_file) > 1: return None # genuinely ambiguous within one file; bail - # Tier 2: same directory. - same_dir = [cid for cid, f in candidate_files.items() - if PurePosixPath(str(f).replace("\\", "/")).parent == call_dir] + # Tier 2: same directory. Two PurePosixPaths are equal exactly when their + # parts match, so comparing the memoized tuples is the same test. + call_parts = _posix_parent_parts(call_norm) + same_dir = [cid for cid, norm in norm_items + if _posix_parent_parts(norm) == call_parts] if len(same_dir) == 1: return same_dir[0] if len(same_dir) > 1: @@ -203,10 +279,9 @@ def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) # Tier 3: longest common path-prefix, computed over path segments. The # winner must be a strict unique maximum, else we bail (guard holds). - call_parts = call_dir.parts - def _common_prefix_len(f: str) -> int: - parts = PurePosixPath(str(f).replace("\\", "/")).parent.parts + def _common_prefix_len(norm: str) -> int: + parts = _posix_parent_parts(norm) n = 0 for a, b in zip(call_parts, parts): if a != b: @@ -215,7 +290,7 @@ def _common_prefix_len(f: str) -> int: return n scored = sorted( - ((cid, _common_prefix_len(f)) for cid, f in candidate_files.items()), + ((cid, _common_prefix_len(norm)) for cid, norm in norm_items), key=lambda kv: kv[1], reverse=True, )