Add a command-line flag and an environment variable to enable pegen (#29

)
python · pablogsal · Apr 22, 2020 · Feb 26, 2020 · Mar 20, 2020 · Mar 20, 2020
commit 33b77af56e50fbc150b704ab48bbdd424f021fe7
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
@@ -252,6 +252,12 @@ Miscellaneous options
    options).  See also :envvar:`PYTHONDEBUG`.
 
 
+.. cmdoption:: -p old|new
+
+   Use the new PEG parser, if arg is ``new``.  If arg is ``old``, use the old
+   parser.  See also :envvar:`PYTHONPARSER`.
+
+
 .. cmdoption:: -E
 
    Ignore all :envvar:`PYTHON*` environment variables, e.g.
@@ -574,6 +580,13 @@ conflict.
    :option:`-d` multiple times.
 
 
+.. envvar:: PYTHONPARSER
+
+   If this is set to ``new``, the new PEG parser is used.  If it is set to
+   ``old``, the old parser is used.  Other values are ignored.  See also the
+   :option:`-p` option.
+
+
 .. envvar:: PYTHONINSPECT
 
    If this is set to a non-empty string it is equivalent to specifying the

diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
@@ -136,6 +136,7 @@ typedef struct {
     int isolated;         /* Isolated mode? see PyPreConfig.isolated */
     int use_environment;  /* Use environment variables? see PyPreConfig.use_environment */
     int dev_mode;         /* Python Development Mode? See PyPreConfig.dev_mode */
+    int use_peg;          /* Use the new PEG-based parser */
 
     /* Install signal handlers? Yes by default. */
     int install_signal_handlers;

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -347,6 +347,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'isolated': 0,
         'use_environment': 1,
         'dev_mode': 0,
+        'use_peg': 0,
 
         'install_signal_handlers': 1,
         'use_hash_seed': 0,
@@ -728,6 +729,7 @@ def test_init_from_config(self):
             'import_time': 1,
             'show_ref_count': 1,
             'malloc_stats': 1,
+            'use_peg': 1,
 
             'stdio_encoding': 'iso8859-1',
             'stdio_errors': 'replace',

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
@@ -486,6 +486,9 @@ static int test_init_from_config(void)
 
     config.install_signal_handlers = 0;
 
+    putenv("PYTHONPARSER=");
+    config.use_peg = 1;
+
     /* FIXME: test use_environment */
 
     putenv("PYTHONHASHSEED=42");

diff --git a/Python/getopt.c b/Python/getopt.c
@@ -41,7 +41,7 @@ static const wchar_t *opt_ptr = L"";
 
 /* Python command line short and long options */
 
-#define SHORT_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?"
+#define SHORT_OPTS L"p:bBc:dEhiIJm:OqRsStuvVW:xX:?"
 
 static const _PyOS_LongOption longopts[] = {
     {L"check-hash-based-pycs", 1, 0},

diff --git a/Python/initconfig.c b/Python/initconfig.c
@@ -37,6 +37,8 @@ Options and arguments (and corresponding environment variables):\n\
 -B     : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\
 -c cmd : program passed in as string (terminates option list)\n\
 -d     : debug output from parser; also PYTHONDEBUG=x\n\
+-p arg : use the new PEG parser, if arg is new; if arg is old, use the old\n\
+         parser; also PYTHONPARSER=arg\n\
 -E     : ignore PYTHON* environment variables (such as PYTHONPATH)\n\
 -h     : print this help message and exit (also --help)\n\
 ";
@@ -642,6 +644,7 @@ config_init_defaults(PyConfig *config)
 
     config->isolated = 0;
     config->use_environment = 1;
+    config->use_peg = 0;
     config->site_import = 1;
     config->bytes_warning = 0;
     config->inspect = 0;
@@ -789,6 +792,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    COPY_ATTR(use_peg);
     COPY_ATTR(install_signal_handlers);
     COPY_ATTR(use_hash_seed);
     COPY_ATTR(hash_seed);
@@ -892,6 +896,7 @@ config_as_dict(const PyConfig *config)
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
     SET_ITEM_INT(dev_mode);
+    SET_ITEM_INT(use_peg);
     SET_ITEM_INT(install_signal_handlers);
     SET_ITEM_INT(use_hash_seed);
     SET_ITEM_UINT(hash_seed);
@@ -1319,6 +1324,16 @@ config_read_env_vars(PyConfig *config)
         config->malloc_stats = 1;
     }
 
+    const char *parser = config_get_env(config, "PYTHONPARSER");
+    if (parser) {
+        if (strcmp(parser, "new") == 0) {
+            config->use_peg = 1;
+        }
+        else if (strcmp(parser, "old") == 0) {
+            config->use_peg = 0;
+        }
+    }
+
     if (config->pythonpath_env == NULL) {
         status = CONFIG_GET_ENV_DUP(config, &config->pythonpath_env,
                                     L"PYTHONPATH", "PYTHONPATH");
@@ -2003,6 +2018,15 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions,
             config->use_hash_seed = 0;
             break;
 
+        case 'p':
+            if (wcscmp(_PyOS_optarg, L"new") == 0) {
+                config->use_peg = 1;
+            }
+            else if (wcscmp(_PyOS_optarg, L"old") == 0) {
+                config->use_peg = 0;
+            }
+            break;
+
         /* This space reserved for other options */
 
         default:
@@ -2505,6 +2529,7 @@ PyConfig_Read(PyConfig *config)
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
     assert(config->dev_mode >= 0);
+    assert(config->use_peg >= 0);
     assert(config->install_signal_handlers >= 0);
     assert(config->use_hash_seed >= 0);
     assert(config->faulthandler >= 0);

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -1023,6 +1023,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     mod_ty mod;
     PyArena *arena;
     PyObject *filename;
+    int use_peg = _PyInterpreterState_GET_UNSAFE()->config.use_peg;
 
     filename = _PyUnicode_FromId(&PyId_string); /* borrowed */
     if (filename == NULL)
@@ -1032,11 +1033,13 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     if (arena == NULL)
         return NULL;
 
-    // I left this here so when you run 'nm python | grep PyPegen' you can see
-    // that the symbol is included and this function is called.
-    mod = PyPegen_ASTFromString("1+1", arena);
+    if (use_peg) {
+        mod = PyPegen_ASTFromString(str, arena);
+    }
+    else {
+        mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+    }
 
-    mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
     if (mod != NULL)
         ret = run_mod(mod, filename, globals, locals, flags, arena);
     PyArena_Free(arena);
@@ -1051,6 +1054,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
     mod_ty mod;
     PyArena *arena = NULL;
     PyObject *filename;
+    int use_peg = _PyInterpreterState_GET_UNSAFE()->config.use_peg;
 
     filename = PyUnicode_DecodeFSDefault(filename_str);
     if (filename == NULL)
@@ -1060,8 +1064,14 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
     if (arena == NULL)
         goto exit;
 
-    mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
-                                     flags, NULL, arena);
+    if (use_peg) {
+        mod = PyPegen_ASTFromFile(filename_str, arena);
+    }
+    else {
+        mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
+                                         flags, NULL, arena);
+    }
+
     if (closeit)
         fclose(fp);
     if (mod == NULL) {