An extended CommonMark compliant parser, with bridges to docutils and Sphinx
—
Advanced rendering system for converting markdown-it tokens to docutils AST nodes, with specialized renderers for docutils and Sphinx environments, HTML processing, and document transformations.
Core rendering functionality for converting markdown-it-py tokens to docutils Abstract Syntax Tree (AST) nodes, providing the foundation for MyST document processing.
class DocutilsRenderer:
"""
Main markdown-it-py renderer for docutils AST.
Converts markdown-it tokens to docutils nodes, handling all standard
MyST syntax elements and providing extension points for custom rendering.
"""
def __init__(self, document) -> None:
"""
Initialize renderer with docutils document.
Args:
document: Docutils document node to populate
"""
def render(self, tokens, options, md_env, renderer=None):
"""
Render markdown-it tokens to docutils nodes.
Args:
tokens: List of markdown-it tokens to render
options: Markdown-it options
md_env: Markdown-it environment
renderer: Optional custom renderer
Returns:
Rendered content (typically None, modifies document in-place)
"""
def create_warning(
self,
message: str,
subtype: MystWarnings = MystWarnings.RENDER_METHOD,
**kwargs
) -> None:
"""
Create and emit a rendering warning.
Args:
message: Warning message
subtype: Warning category
**kwargs: Additional warning context
"""
def add_line_and_source_path(self, node, token) -> None:
"""
Add line number and source path to docutils node.
Args:
node: Docutils node to annotate
token: Source markdown-it token
"""Usage example:
from myst_parser.mdit_to_docutils.base import DocutilsRenderer, make_document
from myst_parser.parsers.mdit import create_md_parser
from myst_parser.config.main import MdParserConfig
# Create document and renderer
document = make_document("example.md")
renderer = DocutilsRenderer(document)
# Create configured parser
config = MdParserConfig(enable_extensions={"tasklist", "deflist"})
md_parser = create_md_parser(config, renderer)
# Parse and render content
source = """
# Document Title
- [x] Completed task
- [ ] Pending task
Term
: Definition of the term
"""
md_parser.render(source)
print(document.pformat())Specialized renderer for Sphinx environments, extending the base DocutilsRenderer with Sphinx-specific functionality and integration features.
class SphinxRenderer(DocutilsRenderer):
"""
Sphinx-specific renderer extending DocutilsRenderer.
Provides Sphinx-specific node handling, cross-reference resolution,
and integration with Sphinx's build system and extensions.
"""
def __init__(self, document) -> None:
"""
Initialize Sphinx renderer.
Args:
document: Sphinx document node to populate
"""
def resolve_reference(self, target: str, reftype: str = None) -> tuple[str, str] | None:
"""
Resolve Sphinx cross-reference.
Args:
target: Reference target
reftype: Type of reference (optional)
Returns:
Tuple of (title, uri) if resolved, None otherwise
"""
def handle_cross_reference(self, token, env) -> None:
"""
Handle cross-reference tokens for Sphinx.
Args:
token: Cross-reference token
env: Markdown-it environment
"""Usage example:
from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
from myst_parser.parsers.mdit import create_md_parser
# In Sphinx environment
def process_myst_document(app, source_path, content):
# Create Sphinx document
document = app.env.new_document(source_path)
# Create Sphinx renderer
renderer = SphinxRenderer(document)
# Create parser with Sphinx configuration
config = create_myst_config(app)
md_parser = create_md_parser(config, renderer)
# Parse with Sphinx-specific handling
md_parser.render(content)
return documentUtility functions for creating and configuring docutils documents with proper settings and metadata.
def make_document(source_path: str, parser_cls=None):
"""
Create new docutils document with proper configuration.
Args:
source_path: Path to source file (for error reporting)
parser_cls: Optional parser class for settings
Returns:
Configured docutils document instance
"""
def token_line(token, default: int | None = None) -> int | None:
"""
Retrieve initial line number of markdown-it token.
Args:
token: Markdown-it token
default: Default line number if not found
Returns:
Line number or default value
"""Usage example:
from myst_parser.mdit_to_docutils.base import make_document, token_line
# Create document for parsing
document = make_document("example.md")
print(document.settings.env)
# Extract line numbers from tokens
def process_tokens(tokens):
for token in tokens:
line_num = token_line(token, default=1)
print(f"Token {token.type} at line {line_num}")Advanced HTML-to-docutils conversion system for handling embedded HTML content within MyST documents.
def html_to_nodes(
text: str,
line_number: int,
renderer: DocutilsRenderer
) -> list[nodes.Element]:
"""
Convert HTML to docutils nodes.
Handles HTML content within MyST documents, supporting HTML images
and admonitions based on enabled extensions.
Args:
text: HTML content to convert
line_number: Source line number for error reporting
renderer: DocutilsRenderer instance for context
Returns:
List of docutils nodes
"""Usage example:
from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes
# Convert HTML to docutils nodes
html_content = """
<div class="custom-container">
<p>HTML content with <em>emphasis</em></p>
<ul>
<li>List item 1</li>
<li>List item 2</li>
</ul>
</div>
"""
# Convert HTML to nodes
nodes = html_to_nodes(
html_content,
line_number=10,
renderer=renderer
)
# Add nodes to document
document.extend(nodes)MyST-Parser integrates with docutils' standard transform system for post-processing rendered documents. Standard docutils transforms handle footnotes, references, and other document-level operations automatically.
Core regular expressions used in rendering:
# URI and link processing
REGEX_SCHEME: Pattern = re.compile(r'^([a-zA-Z][a-zA-Z0-9+.-]*):')
REGEX_URI_TEMPLATE: Pattern = re.compile(r'\{[a-zA-Z_][a-zA-Z0-9_-]*\}')
# Directive processing
REGEX_DIRECTIVE_START: Pattern = re.compile(r'^[\s]{0,3}([`]{3,10}|[~]{3,10}|[:]{3,10})\{')Document rendering related type definitions:
class DocutilsRenderer:
"""Main renderer for markdown-it to docutils conversion."""
class SphinxRenderer(DocutilsRenderer):
"""Sphinx-specific renderer with additional functionality."""
# Type aliases from docutils
nodes.Element # Base docutils node type
nodes.document # Document root nodeInstall with Tessl CLI
npx tessl i tessl/pypi-myst-parser