Skip to content
Navigation Menu
{{ message }}
forked from adafruit/circuitpython
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathextract_pyi.py
More file actions
261 lines (223 loc) · 8.82 KB
/
Copy pathextract_pyi.py
File metadata and controls
261 lines (223 loc) · 8.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT
# Run with 'python tools/extract_pyi.py shared-bindings/ path/to/stub/dir
# You can also test a specific library in shared-bindings by putting the path
# to that directory instead
import ast
import os
import re
import sys
import traceback
import types
import typing
import isort
import black
import circuitpython_typing
import circuitpython_typing.socket
PATHS_IGNORE = frozenset({"shared-bindings/__future__"})
TYPE_MODULE_IMPORTS_IGNORE = frozenset(
{
"array",
"bool",
"buffer",
"bytearray",
"bytes",
"dict",
"file",
"float",
"int",
"list",
"range",
"set",
"slice",
"str",
"struct_time",
"tuple",
}
)
# Include all definitions in these type modules, minus some name conflicts.
AVAILABLE_TYPE_MODULE_IMPORTS = {
"types": frozenset(types.__all__),
# Conflicts: countio.Counter, canio.Match
"typing": frozenset(typing.__all__) - set(["Counter", "Match"]),
"circuitpython_typing": frozenset(circuitpython_typing.__all__),
"circuitpython_typing.socket": frozenset(circuitpython_typing.socket.__all__),
}
def is_typed(node, allow_any=False):
if node is None:
return False
if allow_any:
return True
elif isinstance(node, ast.Name) and node.id == "Any":
return False
elif (
isinstance(node, ast.Attribute)
and type(node.value) is ast.Name
and node.value.id == "typing"
and node.attr == "Any"
):
return False
return True
def find_stub_issues(tree):
for node in ast.walk(tree):
if isinstance(node, ast.AnnAssign):
if not is_typed(node.annotation):
yield ("WARN", f"Missing attribute type on line {node.lineno}")
if isinstance(node.value, ast.Constant) and node.value.value == Ellipsis:
yield ("WARN", f"Unnecessary Ellipsis assignment (= ...) on line {node.lineno}.")
elif isinstance(node, ast.Assign):
if isinstance(node.value, ast.Constant) and node.value.value == Ellipsis:
yield ("WARN", f"Unnecessary Ellipsis assignment (= ...) on line {node.lineno}.")
elif isinstance(node, ast.arguments):
allargs = list(node.args + node.kwonlyargs)
if sys.version_info >= (3, 8):
allargs.extend(node.posonlyargs)
for arg_node in allargs:
if not is_typed(arg_node.annotation) and (
arg_node.arg != "self" and arg_node.arg != "cls"
):
yield (
"WARN",
f"Missing argument type: {arg_node.arg} on line {arg_node.lineno}",
)
if node.vararg and not is_typed(node.vararg.annotation, allow_any=True):
yield (
"WARN",
f"Missing argument type: *{node.vararg.arg} on line {node.vararg.lineno}",
)
if node.kwarg and not is_typed(node.kwarg.annotation, allow_any=True):
yield (
"WARN",
f"Missing argument type: **{node.kwarg.arg} on line {node.kwarg.lineno}",
)
elif isinstance(node, ast.FunctionDef):
if not is_typed(node.returns):
yield ("WARN", f"Missing return type: {node.name} on line {node.lineno}")
def extract_imports(tree):
modules = set()
used_type_module_imports = {module: set() for module in AVAILABLE_TYPE_MODULE_IMPORTS.keys()}
def collect_annotations(anno_tree):
if anno_tree is None:
return
for node in ast.walk(anno_tree):
if isinstance(node, ast.Name):
if node.id in TYPE_MODULE_IMPORTS_IGNORE:
continue
for module, imports in AVAILABLE_TYPE_MODULE_IMPORTS.items():
if node.id in imports:
used_type_module_imports[module].add(node.id)
elif isinstance(node, ast.Attribute):
if isinstance(node.value, ast.Name):
modules.add(node.value.id)
for node in ast.walk(tree):
if isinstance(node, (ast.AnnAssign, ast.arg)):
collect_annotations(node.annotation)
elif isinstance(node, ast.Assign):
collect_annotations(node.value)
elif isinstance(node, ast.FunctionDef):
collect_annotations(node.returns)
for deco in node.decorator_list:
if isinstance(deco, ast.Name) and (
deco.id in AVAILABLE_TYPE_MODULE_IMPORTS["typing"]
):
used_type_module_imports["typing"].add(deco.id)
return (modules, used_type_module_imports)
def find_references(tree):
for node in ast.walk(tree):
if isinstance(node, ast.arguments):
for node in ast.walk(node):
if isinstance(node, ast.Attribute):
if isinstance(node.value, ast.Name) and node.value.id[0].isupper():
yield node.value.id
def convert_folder(top_level, stub_directory):
ok = 0
total = 0
filenames = sorted(os.listdir(top_level))
stub_fragments = []
references = set()
for filename in filenames:
full_path = os.path.join(top_level, filename)
if full_path in PATHS_IGNORE:
continue
file_lines = []
if os.path.isdir(full_path):
(mok, mtotal) = convert_folder(full_path, os.path.join(stub_directory, filename))
ok += mok
total += mtotal
elif filename.endswith(".c"):
with open(full_path, "r", encoding="utf-8") as f:
for line in f:
line = line.rstrip()
if line.startswith("//|"):
if len(line) == 3:
line = ""
elif line[3] == " ":
line = line[4:]
else:
line = line[3:]
print("[WARN] There must be at least one space after '//|'")
file_lines.append(line)
elif filename.endswith(".pyi"):
with open(full_path, "r") as f:
file_lines.extend(line.rstrip() for line in f)
fragment = "\n".join(file_lines).strip()
try:
tree = ast.parse(fragment)
except SyntaxError as e:
print(f"[ERROR] Failed to parse a Python stub from {full_path}")
traceback.print_exception(type(e), e, e.__traceback__)
return (ok, total + 1)
references.update(find_references(tree))
if fragment:
name = os.path.splitext(os.path.basename(filename))[0]
if name == "__init__" or (name in references):
stub_fragments.insert(0, fragment)
else:
stub_fragments.append(fragment)
if not stub_fragments:
return (ok, total)
stub_filename = os.path.join(stub_directory, "__init__.pyi")
print(stub_filename)
stub_contents = "\n\n".join(stub_fragments)
# Validate the stub code.
try:
tree = ast.parse(stub_contents)
except SyntaxError as e:
traceback.print_exception(type(e), e, e.__traceback__)
return (ok, total)
error = False
for level, msg in find_stub_issues(tree):
if level == "ERROR":
error = True
print(f"[{level}] {msg}")
total += 1
if not error:
ok += 1
# Add import statements
imports, type_imports = extract_imports(tree)
import_lines = ["from __future__ import annotations"]
for type_module, used_types in type_imports.items():
if used_types:
import_lines.append(f"from {type_module} import {', '.join(sorted(used_types))}")
import_lines.extend(f"import {m}" for m in sorted(imports))
import_body = "\n".join(import_lines)
m = re.match(r'(\s*""".*?""")', stub_contents, flags=re.DOTALL)
if m:
stub_contents = m.group(1) + "\n\n" + import_body + "\n\n" + stub_contents[m.end() :]
else:
stub_contents = import_body + "\n\n" + stub_contents
# Code formatting
stub_contents = isort.code(stub_contents)
stub_contents = black.format_str(stub_contents, mode=black.FileMode(is_pyi=True))
os.makedirs(stub_directory, exist_ok=True)
with open(stub_filename, "w") as f:
f.write(stub_contents)
return (ok, total)
if __name__ == "__main__":
top_level = sys.argv[1].strip("/")
stub_directory = sys.argv[2]
(ok, total) = convert_folder(top_level, stub_directory)
print(f"Parsing .pyi files: {total - ok} failed, {ok} passed")
if ok != total:
sys.exit(total - ok)
You can’t perform that action at this time.