Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## Unreleased

### 🐛 Bug Fixes

- 🐛 Preserve line-edge whitespace in `parsed-literal` directive content.

## 5.1.0 - 2026-05-13

### ✨ New Features
Expand Down
10 changes: 9 additions & 1 deletion myst_parser/mdit_to_docutils/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from docutils.languages import get_language
from docutils.parsers.rst import Directive, DirectiveError, directives, roles
from docutils.parsers.rst import Parser as RSTParser
from docutils.parsers.rst.directives.body import ParsedLiteral
from docutils.parsers.rst.directives.misc import Include
from docutils.parsers.rst.languages import get_language as get_language_rst
from docutils.statemachine import StringList
Expand Down Expand Up @@ -1898,7 +1899,14 @@ def run_directive(
)
else:
state_machine = MockStateMachine(self, position)
state = MockState(self, state_machine, position)
state = MockState(
self,
state_machine,
position,
preserve_inline_edge_whitespace=issubclass(
directive_class, ParsedLiteral
),
)
directive_instance = directive_class(
name=name,
# the list of positional arguments
Expand Down
63 changes: 55 additions & 8 deletions myst_parser/mocking.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,30 @@
from docutils.parsers.rst.states import Body, Inliner, RSTStateMachine
from docutils.statemachine import StringList
from docutils.utils import unescape
from markdown_it.rules_inline import StateInline

from .parsers.directives import MarkupError, parse_directive_text

if TYPE_CHECKING:
from .mdit_to_docutils.base import DocutilsRenderer


_PRESERVE_LINE_EDGE_WHITESPACE = "myst_preserve_line_edge_whitespace"
_PRESERVE_NEWLINE_RULE = "myst_preserve_newline"


def _preserve_newline(state: StateInline, silent: bool) -> bool:
"""Render a newline without consuming adjacent whitespace."""
if not state.env.get(_PRESERVE_LINE_EDGE_WHITESPACE):
return False
if state.src[state.pos] != "\n":
return False
if not silent:
state.push("softbreak", "br", 0)
state.pos += 1
return True


class MockingError(Exception):
"""An exception to signal an error during mocking of docutils components."""

Expand All @@ -37,6 +54,10 @@ class MockInliner:
def __init__(self, renderer: DocutilsRenderer):
"""Initialize the mock inliner."""
self._renderer = renderer
if _PRESERVE_NEWLINE_RULE not in renderer.md.inline.ruler.get_all_rules():
renderer.md.inline.ruler.before(
"newline", _PRESERVE_NEWLINE_RULE, _preserve_newline
)
# here we mock that the `parse` method has already been called
# which is where these attributes are set (via the RST state Memo)
self.document = renderer.document
Expand All @@ -61,7 +82,13 @@ def problematic(
return problematic

def parse(
self, text: str, lineno: int, memo: Any, parent: nodes.Node
self,
text: str,
lineno: int,
memo: Any,
parent: nodes.Node,
*,
preserve_edge_whitespace: bool = False,
) -> tuple[list[nodes.Node], list[nodes.system_message]]:
"""Parse the text and return a list of nodes."""
# note the only place this is normally called,
Expand All @@ -70,12 +97,23 @@ def parse(
# self.reporter = memo.reporter
# self.document = memo.document
# self.language = memo.language
with self._renderer.current_node_context(parent):
# the parent is never actually appended to though,
# so we make a temporary parent to parse into
container = nodes.Element()
with self._renderer.current_node_context(container):
self._renderer.nested_render_text(text, lineno, inline=True)
previous_preserve = self._renderer.md_env.get(_PRESERVE_LINE_EDGE_WHITESPACE)
if preserve_edge_whitespace:
self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = True
try:
with self._renderer.current_node_context(parent):
# the parent is never actually appended to though,
# so we make a temporary parent to parse into
container = nodes.Element()
with self._renderer.current_node_context(container):
self._renderer.nested_render_text(text, lineno, inline=True)
finally:
if previous_preserve is None:
self._renderer.md_env.pop(_PRESERVE_LINE_EDGE_WHITESPACE, None)
else:
self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = (
previous_preserve
)

return container.children, []

Expand Down Expand Up @@ -104,9 +142,12 @@ def __init__(
renderer: DocutilsRenderer,
state_machine: MockStateMachine,
lineno: int,
*,
preserve_inline_edge_whitespace: bool = False,
):
self._renderer = renderer
self._lineno = lineno
self._preserve_inline_edge_whitespace = preserve_inline_edge_whitespace
self.document = renderer.document
self.reporter = renderer.document.reporter
self.state_machine = state_machine
Expand Down Expand Up @@ -197,7 +238,13 @@ def inline_text(

:returns: (list of nodes, list of messages)
"""
return self.inliner.parse(text, lineno, self.memo, self._renderer.current_node)
return self.inliner.parse(
text,
lineno,
self.memo,
self._renderer.current_node,
preserve_edge_whitespace=self._preserve_inline_edge_whitespace,
)

# U+2014 is an em-dash:
attribution_pattern = re.compile("^((?:---?(?!-)|\u2014) *)(.+)")
Expand Down
51 changes: 51 additions & 0 deletions tests/test_docutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,57 @@ def test_parser():
)


def test_parsed_literal_preserves_line_edge_whitespace():
"""Line-edge whitespace survives inline parsing in parsed literals."""
source = "\n".join(
[
"```{parsed-literal}",
"123",
" 23",
"",
"\t**3** ",
"\ue000 marker",
"```",
"",
]
)
document = publish_doctree(source, parser=Parser())

literal = next(document.findall(nodes.literal_block))
assert literal.astext() == "123\n 23\n\n\t3 \n\ue000 marker"
assert [node.astext() for node in literal.findall(nodes.strong)] == ["3"]


def test_parsed_literal_whitespace_preserves_delimiter_boundaries():
"""Whitespace remains visible to CommonMark delimiter classification."""
source = "\n".join(["```{parsed-literal}", "foo * ", "bar*", "```", ""])
document = publish_doctree(source, parser=Parser())

literal = next(document.findall(nodes.literal_block))
assert literal.astext() == "foo * \nbar*"
assert not list(literal.findall(nodes.emphasis))


def test_parsed_literal_encoded_private_use_entities():
"""Decoded private-use entities are not mistaken for whitespace markers."""
document = publish_doctree(
"```{parsed-literal}\n\n \n```\n", parser=Parser()
)

literal = next(document.findall(nodes.literal_block))
assert literal.astext() == "\ue000\n \ue001"


def test_parsed_literal_all_private_use_characters():
"""Using every private-use character cannot exhaust internal markers."""
private_use = "".join(chr(codepoint) for codepoint in range(0xE000, 0xF900))
source = f"```{{parsed-literal}}\n{private_use}\n indented\n```\n"
document = publish_doctree(source, parser=Parser())

literal = next(document.findall(nodes.literal_block))
assert literal.astext() == f"{private_use}\n indented"


def test_cli_html(monkeypatch, capsys):
monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text")))
cli_html([])
Expand Down