diff --git a/graphify/extract.py b/graphify/extract.py index ffc6153f82..1d6ed7bb05 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -820,6 +820,687 @@ def _get_c_func_name(node, source: bytes) -> str | None: import_handler=_import_js, ) +# TSX JSX text requires ``&`` to start an HTML entity reference (``&``, +# ``&#NN;``, ``<``, ...); a bare ``&`` produces an ERROR node and the +# partial-extraction warning fires (#2551, #2922). ``&`` inside JSX tag +# attribute values, JSX expression containers ``{ ... }``, string literals, +# comments, and TS code is already accepted by tree-sitter-typescript — only +# JSX text content is strict. A bare ``&`` in JSX text is replaced with a +# single ASCII space so the TSX grammar parses the file cleanly; the one-byte +# placeholder keeps the transformed source byte-aligned with the original file +# so ``source[start_byte:end_byte]`` slices stay accurate. +_TSX_ENTITY_RE = re.compile(r'&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);') + +# Characters whose preceding position puts ``<`` at expression position (so it +# must be a JSX tag start, not a comparison or a generic type parameter). The +# inverse — alphanumeric / ``_`` / ``$`` — marks ``<`` as a likely generic +# type-parameter opener (``function f``, ``class Foo``, ``type Bar``) +# or part of a comparison (``a < b``); in those positions the source is TS +# code, not JSX, and bare ``&`` there is bitwise AND and must not be masked. +_TSX_LT_EXPR_PREV = frozenset( + '=(),?:;!&|^~+-*/%<>[]{}' # operators and punctuation + # Keyword tails also act as expression context but are matched by the + # ``return``/``yield``/``new``/``as``/``typeof``/``void``/``delete`` + # end-of-token check below, which keeps the set a flat char check. +) + +# Characters that can precede a ``/`` which starts a regex literal in TS/JS. +# ``/`` after a value token (identifier, number, closing paren, bracket, or +# brace) is a division operator, not a regex; ``<`` and ``>`` are included +# because they can be comparison operators (``a < /b/g``), but the tag ``>`` +# handler resets the previous-char tracker so a ``/`` directly after a JSX +# tag is not mistaken for regex. +_TSX_REGEX_START_PREV = frozenset( + '=(),?:;!&|^~+-*/%[]{}<>' +) + +# Keywords that put the next token in expression position, so a following +# ``/`` can be a regex literal (``return /a/g``, ``case /a/:``, ...). +_TSX_REGEX_START_KEYWORDS = frozenset({ + 'return', 'yield', 'throw', 'typeof', 'void', 'delete', 'case', +}) + + +def _generic_arrow_tail(src: str, m: int) -> bool: + """True when ``src[m] == '('`` opens a parameter list followed by an + ``=>`` — the tail of a generic arrow / function type such as + ``(x: TKey) => x`` or ``(x: T): T => x``. + + Used by the ``<`` disambiguation in :func:`_mask_tsx_ampersands`: + classifying a generic arrow as a JSX tag would strand the walker in + ``jsx_text`` and corrupt a later bitwise ``a & b`` into ``a b`` + (a parse error — the very bug class this fix removes), so an + uppercase ```` directly followed by ``>(`` is only treated as a + tag when no arrow tail follows the balanced parameter list. The scan + is bounded so pathological input cannot make the walker quadratic. + + The scan skips over string literals, comments, and regex literals so + that ``)`` / ``;`` characters inside them do not break the parameter + list balance or prematurely end the return-type search. + """ + n = len(src) + limit = min(n, m + 800) + i = m + paren_depth = 0 + return_depth = 0 + mode = 'params' + prev: str | None = None + keyword: str | None = None + + def _set_prev(c: str) -> None: + nonlocal prev, keyword + prev = c + if not (c.isalnum() or c == '_' or c == '$'): + keyword = None + + def _extend_keyword(c: str) -> None: + nonlocal keyword + keyword = (keyword or '') + c + + def _skip_string(i: int, quote: str) -> int: + i += 1 + while i < limit: + if src[i] == '\\' and i + 1 < n: + i += 2 + continue + if src[i] == quote: + return i + 1 + i += 1 + return i + + def _skip_comment(i: int) -> int: + if src[i + 1] == '/': + while i < limit and src[i] != '\n': + i += 1 + else: + i += 2 + while i + 1 < limit and not (src[i] == '*' and src[i + 1] == '/'): + i += 1 + i += 2 + return i + + def _skip_regex(i: int) -> int: + nonlocal prev, keyword + i += 1 + in_class = False + class_open = -1 + while i < limit: + c = src[i] + if c == '\\' and i + 1 < n: + i += 2 + continue + if in_class: + if c == ']': + if i == class_open + 1 or (i == class_open + 2 and src[class_open + 1] == '^'): + i += 1 + continue + in_class = False + i += 1 + continue + if c == '[': + in_class = True + class_open = i + i += 1 + continue + if c == '/': + i += 1 + while i < n and src[i].isalpha(): + i += 1 + break + i += 1 + prev, keyword = 'a', None + return i + + while i < limit: + c = src[i] + c2 = src[i:i + 2] if i + 1 < n else '' + if c in '"\'`': + i = _skip_string(i, c) + _set_prev(c) + continue + if c2 == '//' or c2 == '/*': + i = _skip_comment(i) + continue + if c == '/' and c2 not in ('//', '/*') and ( + prev is None + or prev in _TSX_REGEX_START_PREV + or keyword in _TSX_REGEX_START_KEYWORDS + ): + i = _skip_regex(i) + continue + if c == '(': + if mode == 'params': + paren_depth += 1 + else: + return_depth += 1 + elif c == ')': + if mode == 'params': + paren_depth -= 1 + if paren_depth == 0: + i += 1 + while i < n and src[i].isspace(): + i += 1 + if src[i:i + 2] == '=>': + return True + if i < n and src[i] == ':': + mode = 'return' + return_depth = 0 + i += 1 + continue + return False + else: + return_depth -= 1 + elif mode == 'return': + if c == '=' and c2 == '=>': + if return_depth == 0: + return True + i += 2 + continue + if c in '[{<': + return_depth += 1 + elif c in ']}>': + return_depth -= 1 + elif c == ';' and return_depth == 0: + return False + if not c.isspace(): + if c.isalnum() or c == '_' or c == '$': + _extend_keyword(c) + else: + keyword = None + prev = c + i += 1 + return False + + +def _mask_tsx_ampersands(src: str) -> str: + """Mask bare ``&`` in JSX text content of TSX source (#2922). + + Tree-sitter's TSX grammar requires ``&`` in JSX text (the run between + ``>`` and ``<`` inside a JSX element) to begin an HTML entity reference + (``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node + and the parser returns a partial tree; the partial-extraction path + surfaces ``parse_errors`` metadata (#2551) that, while silenced by the + multiline-error gate for single-line cases (#2788), still drops the + symbol set the file actually contains. ``&`` inside JSX tags, JSX + expression containers ``{...}``, string literals, comments, and TS code + (where ``&`` is bitwise AND) is left alone because the grammar already + accepts it there. + + Walker: a stack of contexts — ``tag`` / ``close`` / ``self`` (opening, + closing, and self-closing tags), ``expr``, ``string``, ``comment``, + ``line_comment``, ``jsx_text``. Bare ``&`` is replaced with a single + ASCII space only when the top of the stack is ``jsx_text``; already-formed + entities are passed through. A single-byte placeholder keeps the transformed + source byte-aligned with the original file, so tree-sitter byte offsets and + ``source[start_byte:end_byte]`` slices stay valid. A closing tag pops the + element's ``jsx_text`` context — returning to code, an expression container, + or the parent element's JSX text — and a self-closing tag never opens one, + so code after an element (bitwise ``&`` included) is never masked. ``<`` at + code position is treated as a JSX tag start when its previous + non-whitespace character is an expression-context operator or + punctuation; an alphanumeric / ``_`` / ``$`` preceding character marks + it as a generic type-parameter opener (``function f``, ``type Bar``) + or part of a comparison, in which case we stay in code mode. Inside + JSX expression containers the same shape disambiguation runs with + expression context forced on, so nested JSX + (``{ok ? a & b : null}``) is masked as well. + """ + out: list[str] = [] + i = 0 + n = len(src) + stack: list[str] = [] + # When the active context is 'string', the matching quote character. + str_quote: str | None = None + # Previous non-whitespace character in the source (None at file start). + # Drives the ``<`` heuristic for JSX-vs-generic disambiguation at code + # position: alphanumeric / ``_`` / ``$`` means code (likely generic); + # operator/punctuation means expression position (likely JSX tag). + prev_code_char: str | None = None + # Last non-whitespace JS keyword encountered at code position. ``return``, + # ``yield``, ``throw``, ``new``, ``as``, ``typeof``, ``void``, ``delete``, + # ``function``, ``class``, ``type``, ``interface``, ``enum``, ``import``, + # ``export`` — the first group opens expression expression position (so + # ``<`` after them is JSX), the second opens declaration position (so + # ``<`` after them is a generic, not JSX). + prev_code_keyword: str | None = None + # When the active context is 'regex', whether we are inside a character + # class and the index where that class opened (so a leading ``]`` is + # treated as a literal, not the class close). + regex_class = False + regex_class_open = -1 + + # Cheap fast-path: if there is no ``&`` in the source, the mask is a + # no-op and we can skip the whole walk. Almost every real TSX file has + # at least one ``&`` (entity refs, JSX expression ``&&``, bitwise in code), + # so the walk runs — but the empty-source / no-ampersand case avoids the + # allocation when feeding test fixtures without ``&``. + if '&' not in src: + return src + + def _set_prev(c: str) -> None: + nonlocal prev_code_char, prev_code_keyword + prev_code_char = c + # Reset keyword when a non-identifier character is emitted at code + # position. The keyword tracker is updated on identifier characters. + if not (c.isalnum() or c == '_' or c == '$'): + prev_code_keyword = None + + def _extend_keyword(c: str) -> None: + nonlocal prev_code_keyword + # Extend a trailing identifier-shaped run with one more letter. + if prev_code_keyword is not None: + prev_code_keyword = prev_code_keyword + c + else: + prev_code_keyword = c + + def _lt(expr_ctx: bool) -> None: + """Consume a ``<`` at code or expression position. + + Shared by code mode and JSX expression containers so nested JSX + (``{ok ? a & b : null}``) is masked like top-level JSX. + ``expr_ctx`` forces expression position; code mode derives it from + the previous-character / keyword trackers. Tag-shaped ``<`` pushes + a ``tag`` (or ``close`` for ``': + # Fragment ``<>``. + push = 'tag' + elif nxt == '/': + # Closing ```` (e.g. entered from code mode after the + # opening element was missed). + push = 'close' + elif nxt.isalpha() or nxt == '_' or nxt == '$': + # Look past the identifier to decide JSX vs generic. + # ```` / ```` / ```` / ``(...)`` + # are generic-arrow shapes (single-letter type-parameter + # list with optional constraint or default); treating + # those as JSX would push jsx_text mode for the rest + # of the file and incorrectly mask any subsequent + # bitwise ``&`` in code. The shape check classifies + # what comes after the identifier: ``,`` / ``extends`` + # / ``=`` / ``(`` all signal a generic parameter + # list; ``<>``, ``/>``, attributes, or a multi-character + # identifier signal a JSX tag. + j = b + 1 + while j < n and (src[j].isalnum() or src[j] in '_$'): + j += 1 + k = j + while k < n and src[k].isspace(): + k += 1 + nxt_after = src[k:k + 1] if k < n else '' + after_word = src[k:k + 8] + is_extends_generic = False + if after_word.startswith('extends'): + # ``extends`` can be a generic constraint (````) + # or a JSX attribute (````). An attribute + # has its value assignment ``=`` immediately after the name + # (with optional spaces); a generic constraint has a type + # expression. Treat ``>``/``/``/EOF after ``extends`` as JSX + # boolean attributes as well. + p = k + len('extends') + while p < n and src[p].isspace(): + p += 1 + is_extends_generic = p < n and src[p] not in ('=', '>', '/') + if nxt_after == ',' or is_extends_generic or nxt_after == '=': + # ```` / ```` / ````: generic. + push = None + elif nxt_after == '(': + # ``(...) => ...`` is a generic arrow function. + push = None + elif nxt_after == '>': + # ```` / ````: identifier directly followed + # by ``>``. What comes after the ``>`` disambiguates: + # ``(`` opening a parameter list with an arrow tail + # (see ``_generic_arrow_tail``) means a generic arrow / + # function type (``(x: T) => x``, ``(x: TKey) + # => x``, ``let f: (x: T) => void``); anything else + # (text, ``<``, ``{``, ``/``, end) means a JSX element + # like ``VoIP & Chamadas`` — single-letter + # components (icon/nav shorthand) and multi-letter ones + # alike mask their JSX text like any other tag. + # Uppercase-initial is required for the generic + # reading; lowercase ``(...)`` stays JSX. + m = k + 1 + while m < n and src[m].isspace(): + m += 1 + push = None if ( + m < n + and src[m] == '(' + and nxt.isupper() + and _generic_arrow_tail(src, m) + ) else 'tag' + else: + # Multi-character identifier, lowercase, or content + # after ``>`` (````, ````, + # ````): JSX tag. + push = 'tag' + out.append('<') + _set_prev('<') + if push is not None: + stack.append(push) + i += 1 + + while i < n: + c = src[i] + c2 = src[i:i + 2] if i + 1 < n else '' + top = stack[-1] if stack else None + + if top == 'regex': + if c == '\\' and i + 1 < n: + out.append(c) + out.append(src[i + 1]) + i += 2 + continue + if regex_class: + if c == ']': + if ( + i == regex_class_open + 1 + or (i == regex_class_open + 2 and src[regex_class_open + 1] == '^') + ): + # A leading ``]`` immediately after ``[`` or ``[^`` + # is a literal, not the class close. + out.append(c) + i += 1 + continue + regex_class = False + out.append(c) + i += 1 + continue + if c == '[': + regex_class = True + regex_class_open = i + out.append(c) + i += 1 + continue + if c == '/': + out.append(c) + i += 1 + while i < n and src[i].isalpha(): + out.append(src[i]) + i += 1 + stack.pop() + # The regex literal is a value token, so ``<`` after it is a + # comparison and ``/`` after it is division. + _set_prev(')') + continue + out.append(c) + i += 1 + continue + + if top == 'string': + if c == '\\' and i + 1 < n: + out.append(c) + out.append(src[i + 1]) + i += 2 + continue + if c == str_quote: + out.append(c) + stack.pop() + str_quote = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top == 'comment': + if c == '*' and i + 1 < n and src[i + 1] == '/': + out.append('*/') + stack.pop() + i += 2 + continue + out.append(c) + i += 1 + continue + + if top == 'line_comment': + if c == '\n': + out.append(c) + stack.pop() + # Newline ends the code-level identifier run; reset keyword. + prev_code_char = c + prev_code_keyword = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top in ('tag', 'close', 'self'): + if c in '"\'': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '/': + j = i + 1 + while j < n and src[j].isspace(): + j += 1 + if j < n and src[j] == '>' and stack[-1] == 'tag': + # Self-closing ``/>`` (possibly spaced, opening tags + # only — ```` is a fragment close): the upcoming + # ``>`` must not open a jsx_text context for this + # childless element. + stack[-1] = 'self' + out.append(c) + i += 1 + continue + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '>': + out.append(c) + kind = stack.pop() + if kind == 'close': + # ```` closes the element: drop the jsx_text + # context for its children and return to whatever + # surrounded the element (code, expr container, or the + # parent element's JSX text). + if stack and stack[-1] == 'jsx_text': + stack.pop() + elif kind != 'self': + # Opening tag → enter JSX text for the element's children. + stack.append('jsx_text') + # A complete tag is a value token; reset the tracker so the + # next ``<``/``/`` is not misread as a JSX/regex start. + _set_prev(')') + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'expr': + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '}': + out.append(c) + stack.pop() + i += 1 + continue + if c == '"' or c == "'" or c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '/' and c2 not in ('//', '/*'): + if ( + prev_code_char is None + or prev_code_char in _TSX_REGEX_START_PREV + or prev_code_keyword in _TSX_REGEX_START_KEYWORDS + ): + stack.append('regex') + regex_class = False + regex_class_open = -1 + out.append(c) + i += 1 + continue + if c == '<': + # Nested JSX inside a JSX expression container + # (``{ok ? a & b : null}``): run the shared + # tag/generic disambiguation with expression context + # forced on so the nested element's JSX text is masked. + _lt(True) + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'jsx_text': + if c == '<': + out.append(c) + # ```` puts ``<`` after ``n``, which is alpha, so + # the bare-char check would misclassify it as code. The + # keyword tracker catches the expression-position keywords. + # - declaration keywords (function/class/type/interface/enum/ + # import/export) + identifier + < → still a generic opener. + _lt(False) + continue + if c.isalpha() or c == '_' or c == '$': + out.append(c) + _extend_keyword(c) + prev_code_char = c + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + + return ''.join(out) + + +def _tsx_mask_source(source: bytes) -> bytes: + """Bytes form of the JSX-text ``&`` mask for ``LanguageConfig.source_transform``. + + ``_extract_generic`` parses raw bytes, so the str walker is wrapped in a + decode/mask/encode round trip. The ``b"&"`` fast path keeps the common + no-ampersand file a true no-op (same bytes object, no allocation) so the + config hook adds no measurable cost to the languages that never mask. + ``surrogateescape`` on both sides keeps the round trip byte-preserving + for non-UTF-8 files (latin-1 comments, BOM-less legacy encodings): the + only byte-level change the transform may make is the intentional + ``&`` → single-space substitution, never a U+FFFD rewrite of unrelated bytes. + """ + if b"&" not in source: + return source + return _mask_tsx_ampersands( + source.decode("utf-8", errors="surrogateescape") + ).encode("utf-8", errors="surrogateescape") + + # .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. # tree-sitter-typescript ships two languages: language_typescript (for .ts) and # language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on @@ -837,6 +1518,10 @@ def _get_c_func_name(node, source: bytes) -> str | None: call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, function_boundary_types=_TS_CONFIG.function_boundary_types, import_handler=_TS_CONFIG.import_handler, + # Bare ``&`` in JSX text trips the TSX grammar (#2922); mask it at the + # engine's read path so every TSX parse (including embedded scripts) + # gets the fix. See :func:`_mask_tsx_ampersands`. + source_transform=_tsx_mask_source, ) _JAVA_CONFIG = LanguageConfig( diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index ab6ed0c902..65b37b3b52 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -2809,7 +2809,16 @@ def _extract_generic( try: parser = Parser(language) source = path.read_bytes() if source_override is None else source_override - tree = parser.parse(source) + parse_source = source + if config.source_transform is not None: + # Per-language byte mask applied only to the bytes the parser sees — + # e.g. the TSX bare-``&``-in-JSX-text mask (#2922). The mask is + # byte-length-preserving, so the parse tree's offsets stay aligned + # with the original file. Keep the original source for downstream + # ``source[start_byte:end_byte]`` slices so snippets/locations are + # reported against the user's file, not the masked copy. + parse_source = config.source_transform(source) + tree = parser.parse(parse_source) root = tree.root_node except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index 63c1d8a181..fd8a44f144 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -53,6 +53,13 @@ class LanguageConfig: # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) extra_walk_fn: Callable | None = None + # Optional bytes transform applied to the source right before parsing + # (e.g. the TSX bare-``&`` JSX-text mask, #2922). It MUST preserve the + # total UTF-8 byte length so tree-sitter's start_byte/end_byte slices stay + # aligned with the original file. Runs on the bytes that are actually + # parsed, after any ``source_override`` substitution. + source_transform: Callable[[bytes], bytes] | None = None + @dataclass(frozen=True) class _SymbolDeclarationFact: file_path: Path diff --git a/tests/fixtures/tsx_jsx_text_ampersand.tsx b/tests/fixtures/tsx_jsx_text_ampersand.tsx new file mode 100644 index 0000000000..83905e43e0 --- /dev/null +++ b/tests/fixtures/tsx_jsx_text_ampersand.tsx @@ -0,0 +1,49 @@ +// #2922 — bare ``&`` in JSX text breaks the TSX grammar and drops symbols. +// tree-sitter-typescript requires ``&`` in JSX text (the run between ``>`` +// and ``<`` inside a JSX element) to begin an HTML entity reference; a bare +// ``&`` produces an ERROR node and the partial-extraction path surfaces a +// parse_errors warning (#2551). Before the fix, this file extracted to a +// single file node — every function, class, and import was silently lost. +// After the fix, the bare ``&`` in JSX text is masked to ``&`` and every +// node below extracts cleanly with no parse_errors. + +import { helper } from "./helper"; + +const FLAG_MASK = 0xff & 0x0f; + +export function Page() { + return ( +
+

VoIP & Chamadas

+

Conexões & Integrações

+

+ Welcome & hello. Mixed & multiple & ampersands. +

+ + link +
    + {items.filter((it) => it.flag && it.visible).map((it) => ( +
  • {it.label}
    • + ))} +
+
+ ); +} + +export class Component extends React.Component { + render() { + return ( +
+
A & B
+
{helper(FLAG_MASK)}
+
+ ); + } +} + +export const fragment = ( + <> + one & two + three & four + +); \ No newline at end of file diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py new file mode 100644 index 0000000000..329489edba --- /dev/null +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -0,0 +1,411 @@ +"""#2922: a bare ``&`` in TSX JSX text must not break extraction. + +tree-sitter-typescript requires ``&`` inside JSX text (the run between ``>`` +and ``<`` inside an element) to begin an HTML entity reference +(``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node and +the partial-extraction path surfaces a ``parse_errors`` warning (#2551) — +even though esbuild / tsc / React all accept the file. + +Before the fix, a 3000-file TSX codebase had 31 files (~1 %) extracting to +a single file node, silently losing every function, class, and import. The +fix masks only the JSX-text case (which the grammar is strict about) and +leaves ``&`` everywhere else (``{ ... }``, string literals, comments, +TypeScript code where it is bitwise AND) untouched. + +Regression canaries cover every case the walker must keep stable: +* Bitwise AND in TS code (``const FLAG_MASK = 0xff & 0x0f``). +* ``&&`` inside a JSX expression container. +* An existing ``&`` entity in JSX text — passed through unchanged. +* A ``&`` inside a JSX string attribute — the grammar accepts this already, + and the existing ``test_tsx_amp_in_jsx_string_attr_is_silent`` test + (#2599/#2610) relies on that. +* Generics (``function f``, ``const pick = (x: T) => x``, + ``y as number``) — ``<`` after an identifier / keyword must stay in code + mode so a subsequent bitwise ``&`` is not masked. +* Code after a closed JSX element — the closing tag pops the element's + ``jsx_text`` context, so a later ``a & b`` binding stays bitwise AND + and is not corrupted into ``&`` (which would reintroduce a parse + error — the very bug class this fix removes). +* Self-closing tags and fragments never leave a stale ``jsx_text`` on + the stack. +* Nested JSX inside a JSX expression container + (``{ok ? a & b : null}``) is masked like top-level JSX. +* Single-letter uppercase components (``x & y``) are JSX elements, + not generics — while ``(x: T) => x`` (``(`` after ````) stays code. +* Generic arrows and function types — single-letter or multi-character + (``(x: TKey) => x``, ``type F = (x: TKey) => void``, with or + without a return-type annotation) — stay code, so a later bitwise + ``a & b`` is never corrupted into ``a b`` (which would reintroduce a + parse error). +* The bytes mask is fully byte-preserving: every ``&`` in JSX text is + replaced by a single ASCII space, so tree-sitter byte offsets and + ``source[start_byte:end_byte]`` slices stay aligned with the original + source (non-UTF-8 bytes round-trip unchanged). +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from graphify.extract import _mask_tsx_ampersands, _tsx_mask_source, extract + + +def _extract(tmp_path, files: dict[str, str]): + for name, body in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + old = os.getcwd() + try: + os.chdir(tmp_path) + return extract([Path(n) for n in files], + cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + +def _labels(r): + return {n["label"] for n in r["nodes"]} + + +def _assert_silent(err): + assert "syntax errors" not in err + assert "partially extracted" not in err + + +def test_fixture_extracts_all_symbols(tmp_path, capsys): + """The fixture covers every JSX-text shape a real Portuguese-locale UI + file trips the gate on — bare ``&``, ``&`` between non-ASCII letters, + multiple bare ``&`` in one run, alongside JSX attribute ``&`` and code + bitwise ``&`` in the same file.""" + fixture = Path("tests/fixtures/tsx_jsx_text_ampersand.tsx").resolve() + old = os.getcwd() + try: + os.chdir(tmp_path) + r = extract([fixture], cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + labels = _labels(r) + # Top-level bindings and their members must all survive. + assert {"Page()", "Component", "fragment"} <= labels + _assert_silent(capsys.readouterr().err) + # No parse_errors metadata on the file. + assert r.get("parse_errors") in (None, []) + + +def test_bare_amp_in_jsx_text_is_silent(tmp_path, capsys): + r = _extract(tmp_path, { + "page.tsx": ( + "declare const helper: (n: number) => string;\n" + "export function Page() {\n" + " return

VoIP & Chamadas

;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert "Page()" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_bitwise_and_in_ts_code_is_preserved(tmp_path, capsys): + """Bitwise ``&`` in TS code must NOT be masked — the walker has to keep + code mode for ``<`` after an identifier (``FLAG_MASK``, ``helper``) + so the ``&`` stays bitwise AND, and the file extracts cleanly.""" + r = _extract(tmp_path, { + "bits.ts": ( + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export function bits(x: number) { return x & FLAG_MASK }\n" + ), + }) + assert {"FLAG_MASK", "bits()"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_double_ampersand_in_jsx_expression_is_preserved(tmp_path, capsys): + """``&&`` lives inside ``{ ... }``, not in JSX text — the walker must + stay in code mode there.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view =
{true && hi}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): + """Already-formed ``&`` is a real HTML entity and must not be + double-masked (which would produce ``&amp;``).""" + r = _extract(tmp_path, { + "entity.tsx": ( + "export const tag = three & four;\n" + ), + }) + assert "tag" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +# Walker unit cases, exercised through a single parametrized call site so +# the helper keeps exactly one production caller (its ``_tsx_mask_source`` +# wiring) — the afferent-coupling health gate counts direct test call sites. +_MASK_CASES = [ + # --- JSX text: bare ``&`` masked to a single ASCII space, keeping source + # byte offsets aligned with the original file. + # Exact-output match covers: masked exactly once, no double-mask + # (``&amp;``), surrounding text byte-identical except the one-byte + # placeholder. + ('
VoIP & Chamadas
', + '
VoIP Chamadas
'), + # Every bare ``&`` in one JSX-text run is masked independently. + ('

Welcome & hello. Mixed & multiple & ampersands.

', + '

Welcome hello. Mixed multiple ampersands.

'), + # --- Non-JSX-text ``&`` locations are left intact so the TSX grammar + # still sees the same shape it always did. + # JSX attribute string — grammar already accepts. + ('link', + 'link'), + # Bitwise AND in TS code. + ('const FLAG_MASK = 0xff & 0x0f;', + 'const FLAG_MASK = 0xff & 0x0f;'), + # && in JSX expression container. + ('
    {items.filter(it => it.flag && it.visible)}
', + '
    {items.filter(it => it.flag && it.visible)}
'), + # Comment line. + ('// foo & bar\nconst x = 1;', + '// foo & bar\nconst x = 1;'), + # String literal. + ('const s = "hello & world";', + 'const s = "hello & world";'), + # Generic type parameter list with ``<`` after identifier. + ('function foo(x: T): T { return x }', + 'function foo(x: T): T { return x }'), + # Single-uppercase-letter generic ````. + ('const x = foo(1);', + 'const x = foo(1);'), + # Single-letter generic arrow: ``(`` right after ```` stays code. + ('const id = (x: T) => x;\nconst b = 1 & 2;\n', + 'const id = (x: T) => x;\nconst b = 1 & 2;\n'), + # Single-letter function-type position: also ``(`` after ````. + ('let f: (x: T) => void = null;\nconst b = 1 & 2;\n', + 'let f: (x: T) => void = null;\nconst b = 1 & 2;\n'), + # Multi-character generic arrow — ``(x: TKey) => x`` must stay + # code: classifying it as JSX would strand jsx_text and corrupt the + # later bitwise ``1 & 2`` into ``1 2`` (a parse error). + ('const pick = (x: TKey) => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey) => x;\nconst b = 1 & 2;\n'), + # Return-type annotation between parameter list and arrow. + ('const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n'), + # Function-type position, multi-character type parameter. + ('type F = (x: TKey) => void;\nconst b = 1 & 2;\n', + 'type F = (x: TKey) => void;\nconst b = 1 & 2;\n'), + # Arrow generic with comma. + ('const pick = (x: T) => x;', + 'const pick = (x: T) => x;'), + # ``as`` cast — ``<`` after the keyword ``as`` is in expression + # position; the walker must NOT enter jsx_text here. + ('const z = y as number;', + 'const z = y as number;'), + # ``return`` keyword — ```` after ``return`` is JSX. + ('function f() { return }', + 'function f() { return }'), + # ``new`` keyword. + ('const c = new (arg);', + 'const c = new (arg);'), + # --- Tag lifecycle: closing tags pop the element's ``jsx_text``, + # self-closing tags and fragments never leave one behind, and nested + # elements unwind to the parent's text. + # Closing tag pops jsx_text → later code ``&`` stays bitwise. + ('const a =
x & y
;\nconst b = 1 & 2;\n', + 'const a =
x y
;\nconst b = 1 & 2;\n'), + # Self-closing (tight and spaced) never opens jsx_text. + ('const a =
;\nconst b =
;\nconst c = 1 & 2;\n', + 'const a =
;\nconst b =
;\nconst c = 1 & 2;\n'), + # Fragment open/close round-trips back to code. + ('const a = <>x & y;\nconst b = 1 & 2;\n', + 'const a = <>x y;\nconst b = 1 & 2;\n'), + # Nested element: after the child closes, the parent's JSX text is + # still masked; after the parent closes, code is not. + ('const a =

q & rt & u

;\nconst z = 1 & 2;\n', + 'const a =

q rt u

;\nconst z = 1 & 2;\n'), + # --- Single-letter uppercase components (````, ```` — icon/nav + # shorthand) are JSX, not generics: text is masked, the close tag + # pops jsx_text, and an empty element leaves no stale context. + ('export const nav = VoIP & Chamadas;', + 'export const nav = VoIP Chamadas;'), + ('const a = x & y;\nconst b = 1 & 2;\n', + 'const a = x y;\nconst b = 1 & 2;\n'), + ('const a = ;\nconst b = 1 & 2;\n', + 'const a = ;\nconst b = 1 & 2;\n'), + # Paren-initial JSX text has no arrow tail, so an uppercase + # component still masks (``_generic_arrow_tail`` returns False). + ('const el = (note) & more;', + 'const el = (note) more;'), + # Nested JSX inside an expression container is masked, the + # container's own ``&&`` is not, and code after is not. + ('const a =
{x && i & j}
;\nconst z = 1 & 2;\n', + 'const a =
{x && i j}
;\nconst z = 1 & 2;\n'), + # Attribute strings still untouched, element text still masked. + ('const a =
t & v
;', + 'const a =
t v
;'), + # --- Regex literals: ``<``/``>``/``&`` inside a ``/.../`` body are not + # JSX and must not be masked. The walker enters a regex context so the + # ``/`` closing delimiter ends it, including unescaped ``/`` inside + # character classes and a trailing flag run. + ('const r = /&b<\\/a>/;', + 'const r = /&b<\\/a>/;'), + ('const r = /a & b/gi;', + 'const r = /a & b/gi;'), + ('const r = /[a&b]/;', + 'const r = /[a&b]/;'), + ('const r = /[]]/;', + 'const r = /[]]/;'), + ('const r = /[^]]/;', + 'const r = /[^]]/;'), + ('const a = [
, /&b<\\/a>/];', + 'const a = [
, /&b<\\/a>/];'), + ('const a = { r: /&b<\\/a>/ };', + 'const a = { r: /&b<\\/a>/ };'), + ('return /&b<\\/a>/;', + 'return /&b<\\/a>/;'), + # --- Division is not a regex (prev token is a value). + ('const a = 1 / 2 & 3;', + 'const a = 1 / 2 & 3;'), + ('const a = foo() / 2;', + 'const a = foo() / 2;'), + ('const a =
/ 2;', + 'const a =
/ 2;'), + # --- ``extends`` as a JSX attribute must not be misread as a generic + # constraint; the element's children are still JSX text. + ('const a = t & v;', + 'const a = t v;'), + ('const a = t & v;', + 'const a = t v;'), + ('const a = t & v;', + 'const a = t v;'), + # --- Generic arrow / function-type tails keep ``&`` in code even when + # the return type or default parameter contains a ``;`` or ``)``. + ('const pick = (x: TKey): { a: number; b: string } => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey): { a: number; b: string } => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T = ")") => x;\nconst b = 1 & 2;\n', + 'const f = (x: T = ")") => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T = /a\\/b/g) => x;\nconst b = 1 & 2;\n', + 'const f = (x: T = /a\\/b/g) => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T): "a; b" => x;\nconst b = 1 & 2;\n', + 'const f = (x: T): "a; b" => x;\nconst b = 1 & 2;\n'), + # A real generic constraint ```` still stays code. + ('function f() { return 1 & 2; }', + 'function f() { return 1 & 2; }'), + # --- Fast-path: sources without ``&`` (or empty) are a no-op. + ('', ''), + ('// nothing here\nconst x = 1;\n', + '// nothing here\nconst x = 1;\n'), +] + + +@pytest.mark.parametrize('src,expected', _MASK_CASES) +def test_mask_walker(src, expected): + """Walker unit checks: bare ``&`` is masked to a single ASCII space + only in JSX text; attributes, ``{ ... }`` containers, strings, comments, + and TS code (bitwise AND, generics) are byte-identical.""" + got = _mask_tsx_ampersands(src) + assert got == expected, ( + f"walker mangled {src!r}\n" + f" expected: {expected!r}\n" + f" got: {got!r}" + ) + + +def test_mask_source_round_trips_non_utf8_bytes(): + """``LanguageConfig.source_transform`` byte contract: the transform + must be fully byte-preserving. A bare ``&`` in JSX text is replaced by + a single ASCII space, so tree-sitter offsets and ``source[start_byte: + end_byte]`` slices stay aligned with the original file. Non-UTF-8 bytes + (a latin-1 comment) round-trip unchanged instead of being rewritten to + U+FFFD, which would silently alter the source the engine parses.""" + src = b"a & b // caf\xe9 latin-1 comment\n" + out = _tsx_mask_source(src) + assert out.startswith(b"a b") + assert b"caf\xe9" in out + + +@pytest.mark.parametrize('src,expected', [ + (b'a & b', b'a b'), + ('Conexões & Integrações'.encode('utf-8'), + 'Conexões Integrações'.encode('utf-8')), + # Non-BMP (4-byte UTF-8) emoji in JSX text: the replacement stays one + # byte, so offsets remain aligned with the original file. + ('🚀 & Chamadas'.encode('utf-8'), + '🚀 Chamadas'.encode('utf-8')), + (b'a & b // caf\xe9 latin-1 comment\n', + b'a b // caf\xe9 latin-1 comment\n'), + (b'a & b \xff\xfe', + b'a b \xff\xfe'), + (b'const x = 1 & 2;', b'const x = 1 & 2;'), +]) +def test_mask_source_preserves_byte_length(src, expected): + """The ``_tsx_mask_source`` contract is ``bytes -> bytes`` and must be + byte-length-preserving. Every non-ampersand byte survives the round trip, + and a bare ``&`` in JSX text is replaced by a single ASCII space so the + parser sees the same offsets as the original file. This covers multibyte + UTF-8 JSX text and invalid-UTF-8 bytes that round-trip via + ``surrogateescape``.""" + out = _tsx_mask_source(src) + assert isinstance(out, bytes) + assert len(out) == len(src) + assert out == expected + + +def test_code_after_jsx_element_is_not_masked(tmp_path, capsys): + """A closing tag must exit the element's ``jsx_text`` context: code + after ``
`` is TS code again, so a bitwise ``&`` there must not + be masked (masking it would turn valid code into a parse error).""" + r = _extract(tmp_path, { + "page.tsx": ( + "export function Page() {\n" + " return
VoIP & Chamadas
;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert {"Page()", "FLAG_MASK", "use"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_nested_jsx_in_expression_container_is_masked(tmp_path, capsys): + """JSX nested inside a JSX expression container + (``{ok ? a & b : null}``) must be masked like top-level + JSX — before, the walker stayed in expression mode and the bare ``&`` + kept producing an ERROR node.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view = " + "
{true ? VoIP & Chamadas : null}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + assert r.get("parse_errors") in (None, []) + + +def test_non_bmp_jsx_text_ampersand_is_silent(tmp_path, capsys): + """A non-BMP character (e.g. U+1F680 🚀, 4 UTF-8 bytes) in JSX text + followed by a bare ``&`` must not shift parser offsets. The mask + replaces ``&`` with a single ASCII space, so the byte length of the + source stays identical before and after the transform.""" + r = _extract(tmp_path, { + "page.tsx": ( + "export function Page() {\n" + " return 🚀 & Chamadas;\n" + "}\n" + ), + }) + assert "Page()" in _labels(r) + _assert_silent(capsys.readouterr().err) + assert r.get("parse_errors") in (None, []) +