Close #11619: The parser and the import machinery do not encode Unicode · pythoncapi/cpython@14e461d · GitHub
Skip to content

Commit 14e461d

Browse files
vstinnervstinner
committed
Close python#11619: The parser and the import machinery do not encode Unicode
filenames anymore on Windows.
1 parent 33824f6 commit 14e461d

22 files changed

Lines changed: 515 additions & 176 deletions

‎Doc/c-api/exceptions.rst‎

Lines changed: 25 additions & 11 deletions

‎Doc/c-api/veryhigh.rst‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,26 +258,32 @@ the same library that the Python runtime is using.
258258
*optimize* set to ``-1``.
259259
260260
261-
.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize)
261+
.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize)
262262
263263
Parse and compile the Python source code in *str*, returning the resulting code
264264
object. The start token is given by *start*; this can be used to constrain the
265265
code which can be compiled and should be :const:`Py_eval_input`,
266266
:const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by
267267
*filename* is used to construct the code object and may appear in tracebacks or
268-
:exc:`SyntaxError` exception messages, it is decoded from the filesystem
269-
encoding (:func:`sys.getfilesystemencoding`). This returns *NULL* if the
270-
code cannot be parsed or compiled.
268+
:exc:`SyntaxError` exception messages. This returns *NULL* if the code
269+
cannot be parsed or compiled.
271270
272271
The integer *optimize* specifies the optimization level of the compiler; a
273272
value of ``-1`` selects the optimization level of the interpreter as given by
274273
:option:`-O` options. Explicit levels are ``0`` (no optimization;
275274
``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false)
276275
or ``2`` (docstrings are removed too).
277276
278-
.. versionadded:: 3.2
277+
.. versionadded:: 3.4
279278
280279
280+
.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize)
281+
282+
Like :c:func:`Py_CompileStringExFlags`, but *filename* is a byte string
283+
decoded from the filesystem encoding (:func:`os.fsdecode`).
284+
285+
.. versionadded:: 3.2
286+
281287
.. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
282288
283289
This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just

‎Include/ast.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ PyAPI_FUNC(mod_ty) PyAST_FromNode(
1010
PyCompilerFlags *flags,
1111
const char *filename, /* decoded from the filesystem encoding */
1212
PyArena *arena);
13+
PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
14+
const node *n,
15+
PyCompilerFlags *flags,
16+
PyObject *filename,
17+
PyArena *arena);
1318

1419
#ifdef __cplusplus
1520
}

‎Include/compile.h‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,20 @@ PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
3636
PyCompilerFlags *flags,
3737
int optimize,
3838
PyArena *arena);
39-
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
39+
PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject(
40+
struct _mod *mod,
41+
PyObject *filename,
42+
PyCompilerFlags *flags,
43+
int optimize,
44+
PyArena *arena);
45+
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(
46+
struct _mod * mod,
47+
const char *filename /* decoded from the filesystem encoding */
48+
);
49+
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject(
50+
struct _mod * mod,
51+
PyObject *filename
52+
);
4053

4154
/* _Py_Mangle is defined in compile.c */
4255
PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);

‎Include/parsetok.h‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,16 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
4242

4343
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
4444
perrdetail *, int);
45-
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
46-
const char*, grammar *,
47-
int, char *, char *,
48-
perrdetail *, int);
45+
PyAPI_FUNC(node *) PyParser_ParseFileFlags(
46+
FILE *fp,
47+
const char *filename, /* decoded from the filesystem encoding */
48+
const char *enc,
49+
grammar *g,
50+
int start,
51+
char *ps1,
52+
char *ps2,
53+
perrdetail *err_ret,
54+
int flags);
4955
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
5056
FILE *fp,
5157
const char *filename, /* decoded from the filesystem encoding */
@@ -56,18 +62,38 @@ PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
5662
char *ps2,
5763
perrdetail *err_ret,
5864
int *flags);
65+
PyAPI_FUNC(node *) PyParser_ParseFileObject(
66+
FILE *fp,
67+
PyObject *filename,
68+
const char *enc,
69+
grammar *g,
70+
int start,
71+
char *ps1,
72+
char *ps2,
73+
perrdetail *err_ret,
74+
int *flags);
5975

60-
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
61-
const char *,
62-
grammar *, int,
63-
perrdetail *, int);
76+
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(
77+
const char *s,
78+
const char *filename, /* decoded from the filesystem encoding */
79+
grammar *g,
80+
int start,
81+
perrdetail *err_ret,
82+
int flags);
6483
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
6584
const char *s,
6685
const char *filename, /* decoded from the filesystem encoding */
6786
grammar *g,
6887
int start,
6988
perrdetail *err_ret,
7089
int *flags);
90+
PyAPI_FUNC(node *) PyParser_ParseStringObject(
91+
const char *s,
92+
PyObject *filename,
93+
grammar *g,
94+
int start,
95+
perrdetail *err_ret,
96+
int *flags);
7197

7298
/* Note that the following functions are defined in pythonrun.c,
7399
not in parsetok.c */

‎Include/pyerrors.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,9 +301,16 @@ PyAPI_FUNC(void) PyErr_SyntaxLocationEx(
301301
const char *filename, /* decoded from the filesystem encoding */
302302
int lineno,
303303
int col_offset);
304+
PyAPI_FUNC(void) PyErr_SyntaxLocationObject(
305+
PyObject *filename,
306+
int lineno,
307+
int col_offset);
304308
PyAPI_FUNC(PyObject *) PyErr_ProgramText(
305309
const char *filename, /* decoded from the filesystem encoding */
306310
int lineno);
311+
PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject(
312+
PyObject *filename,
313+
int lineno);
307314

308315
/* The following functions are used to create and modify unicode
309316
exceptions from C */

‎Include/pythonrun.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
6666
int start,
6767
PyCompilerFlags *flags,
6868
PyArena *arena);
69+
PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject(
70+
const char *s,
71+
PyObject *filename,
72+
int start,
73+
PyCompilerFlags *flags,
74+
PyArena *arena);
6975
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
7076
FILE *fp,
7177
const char *filename, /* decoded from the filesystem encoding */
@@ -76,6 +82,16 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
7682
PyCompilerFlags *flags,
7783
int *errcode,
7884
PyArena *arena);
85+
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject(
86+
FILE *fp,
87+
PyObject *filename,
88+
const char* enc,
89+
int start,
90+
char *ps1,
91+
char *ps2,
92+
PyCompilerFlags *flags,
93+
int *errcode,
94+
PyArena *arena);
7995
#endif
8096

8197
#ifndef PyParser_SimpleParseString
@@ -117,11 +133,20 @@ PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
117133
int start,
118134
PyCompilerFlags *flags,
119135
int optimize);
136+
PyAPI_FUNC(PyObject *) Py_CompileStringObject(
137+
const char *str,
138+
PyObject *filename, int start,
139+
PyCompilerFlags *flags,
140+
int optimize);
120141
#endif
121142
PyAPI_FUNC(struct symtable *) Py_SymtableString(
122143
const char *str,
123144
const char *filename, /* decoded from the filesystem encoding */
124145
int start);
146+
PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
147+
const char *str,
148+
PyObject *filename,
149+
int start);
125150

126151
PyAPI_FUNC(void) PyErr_Print(void);
127152
PyAPI_FUNC(void) PyErr_PrintEx(int);

‎Include/symtable.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
1616
struct _symtable_entry;
1717

1818
struct symtable {
19-
const char *st_filename; /* name of file being compiled,
19+
PyObject *st_filename; /* name of file being compiled,
2020
decoded from the filesystem encoding */
2121
struct _symtable_entry *st_cur; /* current symbol table entry */
2222
struct _symtable_entry *st_top; /* symbol table entry for module */
@@ -74,6 +74,10 @@ PyAPI_FUNC(struct symtable *) PySymtable_Build(
7474
mod_ty mod,
7575
const char *filename, /* decoded from the filesystem encoding */
7676
PyFutureFeatures *future);
77+
PyAPI_FUNC(struct symtable *) PySymtable_BuildObject(
78+
mod_ty mod,
79+
PyObject *filename,
80+
PyFutureFeatures *future);
7781
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
7882

7983
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);

‎Include/warnings.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ PyAPI_FUNC(int) PyErr_WarnFormat(
1717
Py_ssize_t stack_level,
1818
const char *format, /* ASCII-encoded string */
1919
...);
20+
PyAPI_FUNC(int) PyErr_WarnExplicitObject(
21+
PyObject *category,
22+
PyObject *message,
23+
PyObject *filename,
24+
int lineno,
25+
PyObject *module,
26+
PyObject *registry);
2027
PyAPI_FUNC(int) PyErr_WarnExplicit(
2128
PyObject *category,
2229
const char *message, /* UTF-8 encoded string */

‎Lib/test/test_import.py‎

Lines changed: 9 additions & 1 deletion

0 commit comments

Comments
 (0)