Update grammar and parser · steve-s/graalpython@acea239 · GitHub
Skip to content

Commit acea239

Browse files
otethalotethal
committed
Update grammar and parser
1 parent e441219 commit acea239

57 files changed

Lines changed: 12807 additions & 11383 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎graalpython/com.oracle.graal.python.pegparser.generator/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion

‎graalpython/com.oracle.graal.python.pegparser.generator/input_files/Python.asdl‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
-- Copyright (c) 2021, 2022, Oracle and/or its affiliates.
1+
-- Copyright (c) 2021, 2024, Oracle and/or its affiliates.
22
-- Copyright (C) 1996-2021 Python Software Foundation
33
--
44
-- Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -15,20 +15,22 @@ module Python
1515

1616
stmt = FunctionDef(identifier name, arguments args,
1717
stmt* body, expr* decorator_list, expr? returns,
18-
string? type_comment)
18+
string? type_comment, type_param* type_params)
1919
| AsyncFunctionDef(identifier name, arguments args,
2020
stmt* body, expr* decorator_list, expr? returns,
21-
string? type_comment)
21+
string? type_comment, type_param* type_params)
2222

2323
| ClassDef(identifier name,
2424
expr* bases,
2525
keyword* keywords,
2626
stmt* body,
27-
expr* decorator_list)
27+
expr* decorator_list,
28+
type_param* type_params)
2829
| Return(expr? value)
2930

3031
| Delete(expr* targets)
3132
| Assign(expr* targets, expr value, string? type_comment)
33+
| TypeAlias(expr name, type_param* type_params, expr value)
3234
| AugAssign(expr target, operator op, expr value)
3335
-- 'simple' indicates that we annotate simple name without parens
3436
| AnnAssign(expr target, expr annotation, expr? value, int simple)
@@ -147,4 +149,9 @@ module Python
147149
attributes (int lineno, int col_offset, int end_lineno, int end_col_offset)
148150

149151
type_ignore = TypeIgnore(int lineno, string tag)
152+
153+
type_param = TypeVar(identifier name, expr? bound)
154+
| ParamSpec(identifier name)
155+
| TypeVarTuple(identifier name)
156+
attributes (int lineno, int col_offset, int end_lineno, int end_col_offset)
150157
}

‎graalpython/com.oracle.graal.python.pegparser.generator/input_files/Tokens‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, 2022, Oracle and/or its affiliates.
1+
# Copyright (c) 2021, 2024, Oracle and/or its affiliates.
22
# Copyright (C) 1996-2021 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -58,16 +58,20 @@ ATEQUAL '@='
5858
RARROW '->'
5959
ELLIPSIS '...'
6060
COLONEQUAL ':='
61+
EXCLAMATION '!'
6162

6263
OP
6364
AWAIT
6465
ASYNC
6566
TYPE_IGNORE
6667
TYPE_COMMENT
6768
SOFT_KEYWORD
69+
FSTRING_START
70+
FSTRING_MIDDLE
71+
FSTRING_END
72+
COMMENT
73+
NL
6874
ERRORTOKEN
6975

7076
# These aren't used by the C tokenizer but are needed for tokenize.py
71-
COMMENT
72-
NL
7377
ENCODING

‎graalpython/com.oracle.graal.python.pegparser.generator/input_files/python.gram‎

Lines changed: 117 additions & 29 deletions
Large diffs are not rendered by default.

‎graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py‎

Lines changed: 44 additions & 15 deletions
Large diffs are not rendered by default.

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/test/AtomsTests.java‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -102,7 +102,24 @@ public void atomString7() throws Exception {
102102
@Test
103103
public void atomFString() throws Exception {
104104
checkTreeResult("f'a{b!r}'");
105-
checkError("f'a{b!g}'", "Generic[1:2-1:9]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
105+
checkError("f'a{b!g}'", "Syntax[1:6-1:7]:f-string: invalid conversion character 'g': expected 's', 'r', or 'a'");
106+
}
107+
108+
@Test
109+
public void atomFStringFormatSpec() throws Exception {
110+
checkTreeResult("f'a{b!r:05}'");
111+
}
112+
113+
@Test
114+
public void atomFStringDebug() throws Exception {
115+
checkTreeResult("f'a{b=}'");
116+
}
117+
118+
@Test
119+
public void atomFStringDebugMultiline() throws Exception {
120+
checkTreeResult("f\"\"\"a{b\n" +
121+
"+ # comment\n" +
122+
"c=}\"\"\"");
106123
}
107124

108125
@Test
@@ -114,14 +131,19 @@ public void atomFStringMultiline() throws Exception {
114131
"Another line.\"\"\"");
115132
}
116133

134+
@Test
135+
public void atomFStringGreedy() throws Exception {
136+
checkTreeResult("f\"{1:>3{5}}}}\"");
137+
}
138+
117139
@Test
118140
public void atomByte() throws Exception {
119141
checkTreeResult("b\"a\"");
120142
}
121143

122144
@Test
123145
public void atomMixedBytesString() throws Exception {
124-
checkError("b\"a\" f'aa'", "Generic[1:0-1:10]:cannot mix bytes and nonbytes literals");
146+
checkError("b\"a\" f'aa'", "Syntax[1:10-1:11]:cannot mix bytes and nonbytes literals");
125147
}
126148

127149
@Test
@@ -183,4 +205,19 @@ public void atomEllipsis() throws Exception {
183205
public void atomUnicodeStringPrefix() throws Exception {
184206
checkTreeResult("(u'abc', u'abc' 'def', 'abc' u'def', f'{u\"abc\"}' 'def')");
185207
}
208+
209+
@Test
210+
public void atomMultilineString() throws Exception {
211+
checkTreeResult("'''abc\ndef'''");
212+
}
213+
214+
@Test
215+
public void atomMultilineStringCrLf() throws Exception {
216+
checkTreeResult("'''abc\r\ndef'''");
217+
}
218+
219+
@Test
220+
public void atomMultilineStringCr() throws Exception {
221+
checkTreeResult("'''abc\rdef'''");
222+
}
186223
}

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/test/DefaultStringFactoryImpl.java‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -41,56 +41,59 @@
4141
package com.oracle.graal.python.pegparser.test;
4242

4343
import com.oracle.graal.python.pegparser.PythonStringFactory;
44+
import com.oracle.graal.python.pegparser.sst.ConstantValue;
4445

45-
public class DefaultStringFactoryImpl implements PythonStringFactory<String> {
46+
public class DefaultStringFactoryImpl implements PythonStringFactory {
4647
@Override
47-
public PythonStringBuilder<String> createBuilder(int initialCodePointLength) {
48+
public PythonStringBuilder createBuilder(int initialCodePointLength) {
4849
return new StringBuilderWrapper(initialCodePointLength);
4950
}
5051

5152
@Override
52-
public String emptyString() {
53-
return "";
53+
public ConstantValue fromCodePoints(int[] codepoints, int start, int len) {
54+
return ConstantValue.ofRaw(new String(codepoints, start, len));
5455
}
5556

5657
@Override
57-
public String fromJavaString(String s) {
58-
return s;
58+
public int[] toCodePoints(ConstantValue constantValue) {
59+
return constantValue.getRaw(String.class).codePoints().toArray();
5960
}
6061

61-
private static class StringBuilderWrapper implements PythonStringBuilder<String> {
62+
@Override
63+
public boolean isEmpty(ConstantValue constantValue) {
64+
return constantValue.getRaw(String.class).isEmpty();
65+
}
66+
67+
private static class StringBuilderWrapper implements PythonStringBuilder {
6268
private final StringBuilder sb;
6369

6470
StringBuilderWrapper(int initialCapacity) {
6571
this.sb = new StringBuilder(initialCapacity);
6672
}
6773

6874
@Override
69-
public PythonStringBuilder<String> appendString(String s) {
70-
sb.append(s);
75+
public PythonStringBuilder appendCodePoint(int codePoint) {
76+
sb.appendCodePoint(codePoint);
7177
return this;
7278
}
7379

7480
@Override
75-
public PythonStringBuilder<String> appendPythonString(String s) {
76-
sb.append(s);
77-
return null;
78-
}
79-
80-
@Override
81-
public PythonStringBuilder<String> appendCodePoint(int codePoint) {
82-
sb.appendCodePoint(codePoint);
83-
return null;
81+
public PythonStringBuilder appendCodePoints(int[] codepoints, int offset, int count) {
82+
for (int i = 0; i < count; ++i) {
83+
sb.appendCodePoint(codepoints[offset + i]);
84+
}
85+
return this;
8486
}
8587

8688
@Override
87-
public boolean isEmpty() {
88-
return sb.length() == 0;
89+
public PythonStringBuilder appendConstantValue(ConstantValue constantValue) {
90+
sb.append(constantValue.getRaw(String.class));
91+
return this;
8992
}
9093

9194
@Override
92-
public String build() {
93-
return sb.toString();
95+
public ConstantValue build() {
96+
return ConstantValue.ofRaw(sb.toString());
9497
}
9598
}
9699
}

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/test/FuncDefTests.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -285,37 +285,37 @@ public void positionalOnlyArg06() throws Exception {
285285

286286
@Test
287287
public void positionalOnlyArg07() throws Exception {
288-
checkSyntaxErrorMessage("def name(p1, p2=None, /, p_or_kw, *, kw): pass", "non-default argument follows default argument");
288+
checkSyntaxErrorMessage("def name(p1, p2=None, /, p_or_kw, *, kw): pass", "parameter without a default follows parameter with a default");
289289
}
290290

291291
@Test
292292
public void positionalOnlyArg08() throws Exception {
293-
checkSyntaxErrorMessage("def name(p1=None, p2, /, p_or_kw=None, *, kw): pass", "non-default argument follows default argument");
293+
checkSyntaxErrorMessage("def name(p1=None, p2, /, p_or_kw=None, *, kw): pass", "parameter without a default follows parameter with a default");
294294
}
295295

296296
@Test
297297
public void positionalOnlyArg09() throws Exception {
298-
checkSyntaxErrorMessage("def name(p1=None, p2, /): pass", "non-default argument follows default argument");
298+
checkSyntaxErrorMessage("def name(p1=None, p2, /): pass", "parameter without a default follows parameter with a default");
299299
}
300300

301301
@Test
302302
public void positionalOnlyArg10() throws Exception {
303-
checkSyntaxErrorMessage("def f(a, b = 5, /, c): pass", "non-default argument follows default argument");
303+
checkSyntaxErrorMessage("def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default");
304304
}
305305

306306
@Test
307307
public void positionalOnlyArg11() throws Exception {
308-
checkSyntaxErrorMessage("def f(a = 5, b, /, c): pass", "non-default argument follows default argument");
308+
checkSyntaxErrorMessage("def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default");
309309
}
310310

311311
@Test
312312
public void positionalOnlyArg12() throws Exception {
313-
checkSyntaxErrorMessage("def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument");
313+
checkSyntaxErrorMessage("def f(a = 5, b=1, /, c, *, d=2): pass", "parameter without a default follows parameter with a default");
314314
}
315315

316316
@Test
317317
public void positionalOnlyArg13() throws Exception {
318-
checkSyntaxErrorMessage("def f(a = 5, b, /): pass", "non-default argument follows default argument");
318+
checkSyntaxErrorMessage("def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default");
319319
}
320320

321321
@Test
@@ -395,22 +395,22 @@ public void positionalOnlyArg28() throws Exception {
395395

396396
@Test
397397
public void positionalOnlyArg29() throws Exception {
398-
checkSyntaxErrorMessage("async def f(a, b = 5, /, c): pass", "non-default argument follows default argument");
398+
checkSyntaxErrorMessage("async def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default");
399399
}
400400

401401
@Test
402402
public void positionalOnlyArg30() throws Exception {
403-
checkSyntaxErrorMessage("async def f(a = 5, b, /, c): pass", "non-default argument follows default argument");
403+
checkSyntaxErrorMessage("async def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default");
404404
}
405405

406406
@Test
407407
public void positionalOnlyArg31() throws Exception {
408-
checkSyntaxErrorMessage("async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument");
408+
checkSyntaxErrorMessage("async def f(a = 5, b=1, /, c, d=2): pass", "parameter without a default follows parameter with a default");
409409
}
410410

411411
@Test
412412
public void positionalOnlyArg32() throws Exception {
413-
checkSyntaxErrorMessage("async def f(a = 5, b, /): pass", "non-default argument follows default argument");
413+
checkSyntaxErrorMessage("async def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default");
414414
}
415415

416416
@Test

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/test/LambdaInFunctionTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -107,12 +107,12 @@ public void lambda12() throws Exception {
107107

108108
@Test
109109
public void positionalOnlyArg01() throws Exception {
110-
checkSyntaxErrorMessage("lambda a, b = 5, /, c: None", "non-default argument follows default argument");
110+
checkSyntaxErrorMessage("lambda a, b = 5, /, c: None", "parameter without a default follows parameter with a default");
111111
}
112112

113113
@Test
114114
public void positionalOnlyArg02() throws Exception {
115-
checkSyntaxErrorMessage("lambda a = 5, b, /, c: None", "non-default argument follows default argument");
115+
checkSyntaxErrorMessage("lambda a = 5, b, /, c: None", "parameter without a default follows parameter with a default");
116116
}
117117

118118
@Test

‎graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/test/ParserTestBase.java‎

Lines changed: 8 additions & 6 deletions

0 commit comments

Comments
 (0)