python · vstinner · Nov 2, 2020 · Sep 16, 2020 · Oct 29, 2020 · Oct 29, 2020
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h
@@ -0,0 +1,234 @@
+// File automatically generated by Parser/asdl_c.py.
+
+#ifndef Py_INTERNAL_AST_H
+#define Py_INTERNAL_AST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+struct ast_state {
+    int initialized;
+    PyObject *AST_type;
+    PyObject *Add_singleton;
+    PyObject *Add_type;
+    PyObject *And_singleton;
+    PyObject *And_type;
+    PyObject *AnnAssign_type;
+    PyObject *Assert_type;
+    PyObject *Assign_type;
+    PyObject *AsyncFor_type;
+    PyObject *AsyncFunctionDef_type;
+    PyObject *AsyncWith_type;
+    PyObject *Attribute_type;
+    PyObject *AugAssign_type;
+    PyObject *Await_type;
+    PyObject *BinOp_type;
+    PyObject *BitAnd_singleton;
+    PyObject *BitAnd_type;
+    PyObject *BitOr_singleton;
+    PyObject *BitOr_type;
+    PyObject *BitXor_singleton;
+    PyObject *BitXor_type;
+    PyObject *BoolOp_type;
+    PyObject *Break_type;
+    PyObject *Call_type;
+    PyObject *ClassDef_type;
+    PyObject *Compare_type;
+    PyObject *Constant_type;
+    PyObject *Continue_type;
+    PyObject *Del_singleton;
+    PyObject *Del_type;
+    PyObject *Delete_type;
+    PyObject *DictComp_type;
+    PyObject *Dict_type;
+    PyObject *Div_singleton;
+    PyObject *Div_type;
+    PyObject *Eq_singleton;
+    PyObject *Eq_type;
+    PyObject *ExceptHandler_type;
+    PyObject *Expr_type;
+    PyObject *Expression_type;
+    PyObject *FloorDiv_singleton;
+    PyObject *FloorDiv_type;
+    PyObject *For_type;
+    PyObject *FormattedValue_type;
+    PyObject *FunctionDef_type;
+    PyObject *FunctionType_type;
+    PyObject *GeneratorExp_type;
+    PyObject *Global_type;
+    PyObject *GtE_singleton;
+    PyObject *GtE_type;
+    PyObject *Gt_singleton;
+    PyObject *Gt_type;
+    PyObject *IfExp_type;
+    PyObject *If_type;
+    PyObject *ImportFrom_type;
+    PyObject *Import_type;
+    PyObject *In_singleton;
+    PyObject *In_type;
+    PyObject *Interactive_type;
+    PyObject *Invert_singleton;
+    PyObject *Invert_type;
+    PyObject *IsNot_singleton;
+    PyObject *IsNot_type;
+    PyObject *Is_singleton;
+    PyObject *Is_type;
+    PyObject *JoinedStr_type;
+    PyObject *LShift_singleton;
+    PyObject *LShift_type;
+    PyObject *Lambda_type;
+    PyObject *ListComp_type;
+    PyObject *List_type;
+    PyObject *Load_singleton;
+    PyObject *Load_type;
+    PyObject *LtE_singleton;
+    PyObject *LtE_type;
+    PyObject *Lt_singleton;
+    PyObject *Lt_type;
+    PyObject *MatMult_singleton;
+    PyObject *MatMult_type;
+    PyObject *Mod_singleton;
+    PyObject *Mod_type;
+    PyObject *Module_type;
+    PyObject *Mult_singleton;
+    PyObject *Mult_type;
+    PyObject *Name_type;
+    PyObject *NamedExpr_type;
+    PyObject *Nonlocal_type;
+    PyObject *NotEq_singleton;
+    PyObject *NotEq_type;
+    PyObject *NotIn_singleton;
+    PyObject *NotIn_type;
+    PyObject *Not_singleton;
+    PyObject *Not_type;
+    PyObject *Or_singleton;
+    PyObject *Or_type;
+    PyObject *Pass_type;
+    PyObject *Pow_singleton;
+    PyObject *Pow_type;
+    PyObject *RShift_singleton;
+    PyObject *RShift_type;
+    PyObject *Raise_type;
+    PyObject *Return_type;
+    PyObject *SetComp_type;
+    PyObject *Set_type;
+    PyObject *Slice_type;
+    PyObject *Starred_type;
+    PyObject *Store_singleton;
+    PyObject *Store_type;
+    PyObject *Sub_singleton;
+    PyObject *Sub_type;
+    PyObject *Subscript_type;
+    PyObject *Try_type;
+    PyObject *Tuple_type;
+    PyObject *TypeIgnore_type;
+    PyObject *UAdd_singleton;
+    PyObject *UAdd_type;
+    PyObject *USub_singleton;
+    PyObject *USub_type;
+    PyObject *UnaryOp_type;
+    PyObject *While_type;
+    PyObject *With_type;
+    PyObject *YieldFrom_type;
+    PyObject *Yield_type;
+    PyObject *__dict__;
+    PyObject *__doc__;
+    PyObject *__module__;
+    PyObject *_attributes;
+    PyObject *_fields;
+    PyObject *alias_type;
+    PyObject *annotation;
+    PyObject *arg;
+    PyObject *arg_type;
+    PyObject *args;
+    PyObject *argtypes;
+    PyObject *arguments_type;
+    PyObject *asname;
+    PyObject *ast;
+    PyObject *attr;
+    PyObject *bases;
+    PyObject *body;
+    PyObject *boolop_type;
+    PyObject *cause;
+    PyObject *cmpop_type;
+    PyObject *col_offset;
+    PyObject *comparators;
+    PyObject *comprehension_type;
+    PyObject *context_expr;
+    PyObject *conversion;
+    PyObject *ctx;
+    PyObject *decorator_list;
+    PyObject *defaults;
+    PyObject *elt;
+    PyObject *elts;
+    PyObject *end_col_offset;
+    PyObject *end_lineno;
+    PyObject *exc;
+    PyObject *excepthandler_type;
+    PyObject *expr_context_type;
+    PyObject *expr_type;
+    PyObject *finalbody;
+    PyObject *format_spec;
+    PyObject *func;
+    PyObject *generators;
+    PyObject *handlers;
+    PyObject *id;
+    PyObject *ifs;
+    PyObject *is_async;
+    PyObject *items;
+    PyObject *iter;
+    PyObject *key;
+    PyObject *keys;
+    PyObject *keyword_type;
+    PyObject *keywords;
+    PyObject *kind;
+    PyObject *kw_defaults;
+    PyObject *kwarg;
+    PyObject *kwonlyargs;
+    PyObject *left;
+    PyObject *level;
+    PyObject *lineno;
+    PyObject *lower;
+    PyObject *mod_type;
+    PyObject *module;
+    PyObject *msg;
+    PyObject *name;
+    PyObject *names;
+    PyObject *op;
+    PyObject *operand;
+    PyObject *operator_type;
+    PyObject *ops;
+    PyObject *optional_vars;
+    PyObject *orelse;
+    PyObject *posonlyargs;
+    PyObject *returns;
+    PyObject *right;
+    PyObject *simple;
+    PyObject *slice;
+    PyObject *step;
+    PyObject *stmt_type;
+    PyObject *tag;
+    PyObject *target;
+    PyObject *targets;
+    PyObject *test;
+    PyObject *type;
+    PyObject *type_comment;
+    PyObject *type_ignore_type;
+    PyObject *type_ignores;
+    PyObject *unaryop_type;
+    PyObject *upper;
+    PyObject *value;
+    PyObject *values;
+    PyObject *vararg;
+    PyObject *withitem_type;
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_AST_H */
+
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
@@ -8,10 +8,11 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atomic.h"    /* _Py_atomic_address */
-#include "pycore_gil.h"       /* struct _gil_runtime_state  */
-#include "pycore_gc.h"        /* struct _gc_runtime_state */
-#include "pycore_warnings.h"  /* struct _warnings_runtime_state */
+#include "pycore_atomic.h"        // _Py_atomic_address
+#include "pycore_ast.h"           // struct ast_state
+#include "pycore_gil.h"           // struct _gil_runtime_state
+#include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_warnings.h"      // struct _warnings_runtime_state
 
 struct _pending_calls {
     PyThread_type_lock lock;
@@ -258,6 +259,8 @@ struct _is {
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;
     struct _Py_exc_state exc_state;
+
+    struct ast_state ast;
 };
 
 /* Used by _PyImport_Cleanup() */

diff --git a/Makefile.pre.in b/Makefile.pre.in
@@ -840,9 +840,11 @@ regen-ast:
 	$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
 		$(srcdir)/Parser/Python.asdl \
 		-H $(srcdir)/Include/Python-ast.h.new \
+		-I $(srcdir)/Include/internal/pycore_ast.h.new \
 		-C $(srcdir)/Python/Python-ast.c.new
 
 	$(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
+	$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new
 	$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
 
 .PHONY: regen-opcode

diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst
@@ -0,0 +1,2 @@
+The :mod:`ast` module internal state is now per interpreter. Patch by Victor
+Stinner.
diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj
@@ -137,6 +137,8 @@
     </None>
     <None Include="..\Include\opcode.h">
     </None>
+    <None Include="..\Include\internal\pycore_ast.h">
+    </None>
     <None Include="..\Include\Python-ast.h">
     </None>
     <None Include="..\Python\Python-ast.c">
@@ -161,15 +163,18 @@
     <Warning Text="Pegen updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedParse)' != ''" />
   </Target>
   <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
-    <!-- Regenerate Include/Python-ast.h and Python/Python-ast.c using Parser/asdl_c.py -h -->
-    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot;" />
+    <!-- Regenerate Include/Python-ast.h, Python/Python-ast.c and Include/internal/pycore_ast.h using Parser/asdl_c.py -h -->
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot; -H &quot;$(IntDir)Python-ast.h&quot; -C &quot;$(IntDir)Python-ast.c&quot; -I &quot;$(IntDir)pycore_ast.h&quot;" />
     <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
       <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
     </Copy>
     <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
       <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
     </Copy>
-    <Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+    <Copy SourceFiles="$(IntDir)pycore_ast.h" DestinationFiles="$(PySourcePath)Include\internal\pycore_ast.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedInternalH" />
+    </Copy>
+    <Warning Text="ASDL is updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' != '' and '@(_UpdatedC)' != '' and @(_UpdatedInternalH)'" />
   </Target>
   <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
     <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
@@ -204,6 +209,7 @@
       <Clean Include="$(IntDir)opcode.h" />
       <Clean Include="$(IntDir)Python-ast.c" />
       <Clean Include="$(IntDir)Python-ast.h" />
+      <Clean Include="$(IntDir)pycore_ast.h" />
     </ItemGroup>
   </Target>
 </Project>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		The :mod:`ast` module internal state is now per interpreter. Patch by Victor
		Stinner.