bpo-40939: Clean and adapt the peg_generator directory after deleting the old parser (GH-20822)

Author: pablogsal

Tools/peg_generator/Makefile

@@ -22,7 +22,7 @@ data/xxl.py:
 
 build: peg_extension/parse.c
 
-peg_extension/parse.c: $(GRAMMAR) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h pegen/grammar_parser.py
+peg_extension/parse.c: $(GRAMMAR) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen.c ../../Parser/string_parser.c ../../Parser/*.h pegen/grammar_parser.py
 	$(PYTHON) -m pegen -q c $(GRAMMAR) $(TOKENS) -o peg_extension/parse.c --compile-extension
 
 clean:
@@ -70,18 +70,10 @@ stats: peg_extension/parse.c data/xxl.py
 time: time_compile
 
 time_compile: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl compile
+	$(VENVPYTHON) scripts/benchmark.py --target=xxl compile
 
 time_parse: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl parse
-
-time_old: time_old_compile
-
-time_old_compile: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl compile
-
-time_old_parse: venv data/xxl.py
-	$(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl parse
+	$(VENVPYTHON) scripts/benchmark.py --target=xxl parse
 
 time_peg_dir: venv
 	$(VENVPYTHON) scripts/test_parse_directory.py \

Tools/peg_generator/pegen/keywordgen.py

@@ -41,9 +41,7 @@ def main():
         "grammar", type=str, help="The file with the grammar definition in PEG format"
     )
     parser.add_argument(
-        "tokens_file",
-        type=argparse.FileType("r"),
-        help="The file with the token definitions"
+        "tokens_file", type=argparse.FileType("r"), help="The file with the token definitions"
     )
     parser.add_argument(
         "keyword_file",
@@ -61,9 +59,7 @@ def main():
     gen.collect_todo()
 
     with args.keyword_file as thefile:
-        all_keywords = sorted(
-            list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS
-        )
+        all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS)
 
         keywords = ",\n    ".join(map(repr, all_keywords))
         thefile.write(TEMPLATE.format(keywords=keywords))

Tools/peg_generator/scripts/benchmark.py

@@ -6,13 +6,13 @@
 import os
 from time import time
 
-import _peg_parser
-
 try:
     import memory_profiler
 except ModuleNotFoundError:
-    print("Please run `make venv` to create a virtual environment and install"
-          " all the dependencies, before running this script.")
+    print(
+        "Please run `make venv` to create a virtual environment and install"
+        " all the dependencies, before running this script."
+    )
     sys.exit(1)
 
 sys.path.insert(0, os.getcwd())
@@ -21,13 +21,6 @@
 argparser = argparse.ArgumentParser(
     prog="benchmark", description="Reproduce the various pegen benchmarks"
 )
-argparser.add_argument(
-    "--parser",
-    action="store",
-    choices=["new", "old"],
-    default="pegen",
-    help="Which parser to benchmark (default is pegen)",
-)
 argparser.add_argument(
     "--target",
     action="store",
@@ -40,12 +33,7 @@
 command_compile = subcommands.add_parser(
     "compile", help="Benchmark parsing and compiling to bytecode"
 )
-command_parse = subcommands.add_parser(
-    "parse", help="Benchmark parsing and generating an ast.AST"
-)
-command_notree = subcommands.add_parser(
-    "notree", help="Benchmark parsing and dumping the tree"
-)
+command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST")
 
 
 def benchmark(func):
@@ -66,59 +54,37 @@ def wrapper(*args):
 
 
 @benchmark
-def time_compile(source, parser):
-    if parser == "old":
-        return _peg_parser.compile_string(
-            source,
-            oldparser=True,
-        )
-    else:
-        return _peg_parser.compile_string(source)
-
-
-@benchmark
-def time_parse(source, parser):
-    if parser == "old":
-        return _peg_parser.parse_string(source, oldparser=True)
-    else:
-        return _peg_parser.parse_string(source)
+def time_compile(source):
+    return compile(source, "<string>", "exec")
 
 
 @benchmark
-def time_notree(source, parser):
-    if parser == "old":
-        return _peg_parser.parse_string(source, oldparser=True, ast=False)
-    else:
-        return _peg_parser.parse_string(source, ast=False)
+def time_parse(source):
+    return ast.parse(source)
 
 
-def run_benchmark_xxl(subcommand, parser, source):
+def run_benchmark_xxl(subcommand, source):
     if subcommand == "compile":
-        time_compile(source, parser)
+        time_compile(source)
     elif subcommand == "parse":
-        time_parse(source, parser)
-    elif subcommand == "notree":
-        time_notree(source, parser)
+        time_parse(source)
 
 
-def run_benchmark_stdlib(subcommand, parser):
-    modes = {"compile": 2, "parse": 1, "notree": 0}
+def run_benchmark_stdlib(subcommand):
+    modes = {"compile": 2, "parse": 1}
     for _ in range(3):
         parse_directory(
             "../../Lib",
             verbose=False,
             excluded_files=["*/bad*", "*/lib2to3/tests/data/*",],
-            tree_arg=0,
             short=True,
             mode=modes[subcommand],
-            oldparser=(parser == "old"),
         )
 
 
 def main():
     args = argparser.parse_args()
     subcommand = args.subcommand
-    parser = args.parser
     target = args.target
 
     if subcommand is None:
@@ -127,9 +93,9 @@ def main():
     if target == "xxl":
         with open(os.path.join("data", "xxl.py"), "r") as f:
             source = f.read()
-            run_benchmark_xxl(subcommand, parser, source)
+            run_benchmark_xxl(subcommand, source)
     elif target == "stdlib":
-        run_benchmark_stdlib(subcommand, parser)
+        run_benchmark_stdlib(subcommand)
 
 
 if __name__ == "__main__":

Tools/peg_generator/scripts/find_max_nesting.py

@@ -14,8 +14,7 @@
 Usage: python -m scripts.find_max_nesting
 """
 import sys
-
-from _peg_parser import parse_string
+import ast
 
 GRAMMAR_FILE = "data/python.gram"
 INITIAL_NESTING_DEPTH = 10
@@ -28,9 +27,8 @@
 
 def check_nested_expr(nesting_depth: int) -> bool:
     expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
-
     try:
-        parse_string(expr)
+        ast.parse(expr)
         print(f"Nesting depth of {nesting_depth} is successful")
         return True
     except Exception as err: