From 1c735d059c74ee26c869284ce323b15962aa518b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:57:17 +0100 Subject: [PATCH 1/5] Infra: Add ASDL lexer from cpython --- pep_sphinx_extensions/lexers/asdl_lexer.py | 53 ++++++++++++++++++++++ peps/pep-0823.rst | 2 +- peps/pep-0824.rst | 4 +- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 pep_sphinx_extensions/lexers/asdl_lexer.py diff --git a/pep_sphinx_extensions/lexers/asdl_lexer.py b/pep_sphinx_extensions/lexers/asdl_lexer.py new file mode 100644 index 00000000000..a2f4fb93afb --- /dev/null +++ b/pep_sphinx_extensions/lexers/asdl_lexer.py @@ -0,0 +1,53 @@ +# The code in this file is based on: +# https://github.com/python/cpython/blob/v3.15.0/Doc/tools/extensions/lexers/asdl_lexer.py + +# Licensed under the Python Software Foundation License v2 +# For the full license text, see: +# https://github.com/python/cpython/blob/v3.15.0/LICENSE + +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text + + +class ASDLLexer(RegexLexer): + name = "ASDL" + aliases = ["asdl"] + filenames = ["*.asdl"] + _name = r"([^\W\d]\w*)" + _text_ws = r"(\s*)" + + tokens = { + "ws": [ + (r"\n", Text), + (r"\s+", Text), + (r"--.*?$", Comment.Singleline), + ], + "root": [ + include("ws"), + ( + r"(module)" + _text_ws + _name, + bygroups(Keyword, Text, Name.Tag), + ), + ( + r"(\w+)([\?\*]*\s)(\w+)", + bygroups(Name.Builtin.Pseudo, Operator, Name), + ), + # Keep in line with ``builtin_types`` from Parser/asdl.py. + # ASDL's 4 builtin types are + # constant, identifier, int, string + ("constant|identifier|int|string", Name.Builtin), + (r"attributes", Name.Builtin), + ( + _name + _text_ws + "(=)", + bygroups(Name, Text, Operator), + ), + (_name, Name.Class), + (r"\|", Operator), + (r"{|}|\(|\)", Punctuation), + (r".", Text), + ], + } + + +def register(): + return [ASDLLexer] diff --git a/peps/pep-0823.rst b/peps/pep-0823.rst index 78198a86f39..64c39bb1cd4 100644 --- a/peps/pep-0823.rst +++ b/peps/pep-0823.rst @@ -465,7 +465,7 @@ do not support assignments themselves and thus the context will always be expression nodes. It is set to ``1`` if an expression is the topmost node in a group, ``0`` otherwise. -:: +.. code-block:: ASDL expr = ... | Attribute(expr value, identifier attr, expr_context ctx) diff --git a/peps/pep-0824.rst b/peps/pep-0824.rst index c9989050a19..c6cd257133b 100644 --- a/peps/pep-0824.rst +++ b/peps/pep-0824.rst @@ -218,7 +218,7 @@ AST changes A new ``Coalesce`` operator is added to ``boolop`` for use in ``BoolOp`` nodes. -:: +.. code-block:: ASDL expr = BoolOp(boolop op, expr* values) | ... @@ -284,7 +284,7 @@ A new ``BoolAssign`` AST node is added. Similar to ``AugAssign``, it stores a ``target`` and ``value`` expression as well as the boolean operator ``Coalesce``. -:: +.. code-block:: ASDL stmt = ... | AugAssign(expr target, operator op, expr value) From b501e1336679fa6c921bdc44de935a5b3e402218 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:50:35 +0100 Subject: [PATCH 2/5] Add note to license file --- pep_sphinx_extensions/LICENCE.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pep_sphinx_extensions/LICENCE.rst b/pep_sphinx_extensions/LICENCE.rst index d68de666acb..e9427f5eb56 100644 --- a/pep_sphinx_extensions/LICENCE.rst +++ b/pep_sphinx_extensions/LICENCE.rst @@ -1,2 +1,6 @@ The files in this directory are placed in the public domain or under the CC0-1.0-Universal license, whichever is more permissive. + +Some files are based on work in other projects. Check the individual +file header for up to date information. In particular: +- lexers/asdl_lexer.py From ba4c565bb5562b551ace18e8a9b17c8522bfad33 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:07:46 +0100 Subject: [PATCH 3/5] Minor --- pep_sphinx_extensions/LICENCE.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pep_sphinx_extensions/LICENCE.rst b/pep_sphinx_extensions/LICENCE.rst index e9427f5eb56..af8b9980ba3 100644 --- a/pep_sphinx_extensions/LICENCE.rst +++ b/pep_sphinx_extensions/LICENCE.rst @@ -1,5 +1,6 @@ The files in this directory are placed in the public domain or under the -CC0-1.0-Universal license, whichever is more permissive. +CC0-1.0-Universal license unless otherwise stated, whichever is more +permissive. Some files are based on work in other projects. Check the individual file header for up to date information. In particular: From be619913eb972c49b6c1e089ac93e5c019849d31 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:47:15 +0100 Subject: [PATCH 4/5] This reads better --- pep_sphinx_extensions/LICENCE.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pep_sphinx_extensions/LICENCE.rst b/pep_sphinx_extensions/LICENCE.rst index af8b9980ba3..4f771d4540b 100644 --- a/pep_sphinx_extensions/LICENCE.rst +++ b/pep_sphinx_extensions/LICENCE.rst @@ -1,7 +1,7 @@ -The files in this directory are placed in the public domain or under the -CC0-1.0-Universal license unless otherwise stated, whichever is more +Unless otherwise stated, the files in this directory are placed in the +public domain or under the CC0-1.0-Universal license, whichever is more permissive. -Some files are based on work in other projects. Check the individual -file header for up to date information. In particular: +Some files are based on work from external projects. Check the individual +file headers for specific licensing details. In particular: - lexers/asdl_lexer.py From 1dec1f73b83a50409ca749b86c2d0dd2176b1d45 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:21:21 +0200 Subject: [PATCH 5/5] Link to latest stable tag --- pep_sphinx_extensions/lexers/asdl_lexer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pep_sphinx_extensions/lexers/asdl_lexer.py b/pep_sphinx_extensions/lexers/asdl_lexer.py index a2f4fb93afb..62928a86698 100644 --- a/pep_sphinx_extensions/lexers/asdl_lexer.py +++ b/pep_sphinx_extensions/lexers/asdl_lexer.py @@ -1,9 +1,9 @@ # The code in this file is based on: -# https://github.com/python/cpython/blob/v3.15.0/Doc/tools/extensions/lexers/asdl_lexer.py +# https://github.com/python/cpython/blob/v3.15.2/Doc/tools/extensions/lexers/asdl_lexer.py # Licensed under the Python Software Foundation License v2 # For the full license text, see: -# https://github.com/python/cpython/blob/v3.15.0/LICENSE +# https://github.com/python/cpython/blob/v3.15.2/LICENSE from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Comment, Keyword, Name, Operator, Punctuation, Text