{{ message }}
forked from matplotlib/matplotlib
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcheck_typehints.py
More file actions
executable file
·319 lines (279 loc) · 10.8 KB
/
Copy pathcheck_typehints.py
File metadata and controls
executable file
·319 lines (279 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
#!/usr/bin/env python
"""
Perform AST checks to validate consistency of type hints with implementation.
NOTE: in most cases ``stubtest`` (distributed as part of ``mypy``) should be preferred
This script was written before the configuration of ``stubtest`` was well understood.
It still has some utility, particularly for checking certain deprecations or other
decorators which modify the runtime signature where you want the type hint to match
the python source rather than runtime signature, perhaps.
The basic kinds of checks performed are:
- Set of items defined by the stubs vs implementation
- Missing stub: MISSING_STUB = 1
- Missing implementation: MISSING_IMPL = 2
- Signatures of functions/methods
- Positional Only Args: POS_ARGS = 4
- Keyword or Positional Args: ARGS = 8
- Variadic Positional Args: VARARG = 16
- Keyword Only Args: KWARGS = 32
- Variadic Keyword Only Args: VARKWARG = 64
There are some exceptions to when these are checked:
- Set checks (MISSING_STUB/MISSING_IMPL) only apply at the module level
- i.e. not for classes
- Inheritance makes the set arithmetic harder when only loading AST
- Attributes also make it more complicated when defined in init
- Functions type hinted with ``overload`` are ignored for argument checking
- Usually this means the implementation is less strict in signature but will raise
if an invalid signature is used, type checking allows such errors to be caught by
the type checker instead of at runtime.
- Private attribute/functions are ignored
- Not expecting a type hint
- applies to anything beginning, but not ending in ``__``
- If ``__all__`` is defined, also applies to anything not in ``__all__``
- Deprecated methods are not checked for missing stub
- Only applies to wholesale deprecation, not deprecation of an individual arg
- Other kinds of deprecations (e.g. argument deletion or rename) the type hint should
match the current python definition line still.
- For renames, the new name is used
- For deletions/make keyword only, it is removed upon expiry
Usage:
Currently there is not any argument handling/etc, so all configuration is done in
source.
Since stubtest has almost completely superseded this script, this is unlikely to change.
The categories outlined above can each be ignored, and ignoring multiple can be done
using the bitwise or (``|``) operator, e.g. ``ARGS | VARKWARG``.
This can be done globally or on a per file basis, by editing ``per_file_ignore``.
For the latter, the key is the Pathlib Path to the affected file, and the value is the
integer ignore.
Must be run from repository root:
``python tools/check_typehints.py``
"""
import ast
import pathlib
import sys
MISSING_STUB = 1
MISSING_IMPL = 2
POS_ARGS = 4
ARGS = 8
VARARG = 16
KWARGS = 32
VARKWARG = 64
def check_file(path, ignore=0):
stubpath = path.with_suffix(".pyi")
ret = 0
if not stubpath.exists():
return 0, 0
tree = ast.parse(path.read_text())
stubtree = ast.parse(stubpath.read_text())
return check_namespace(tree, stubtree, path, ignore)
def check_namespace(tree, stubtree, path, ignore=0):
ret = 0
count = 0
tree_items = set(
i.name
for i in tree.body
if hasattr(i, "name") and (not i.name.startswith("_") or i.name.endswith("__"))
)
stubtree_items = set(
i.name
for i in stubtree.body
if hasattr(i, "name") and (not i.name.startswith("_") or i.name.endswith("__"))
)
for item in tree.body:
if isinstance(item, ast.Assign):
tree_items |= set(
i.id
for i in item.targets
if hasattr(i, "id")
and (not i.id.startswith("_") or i.id.endswith("__"))
)
for target in item.targets:
if isinstance(target, ast.Tuple):
tree_items |= set(i.id for i in target.elts)
elif isinstance(item, ast.AnnAssign):
tree_items |= {item.target.id}
for item in stubtree.body:
if isinstance(item, ast.Assign):
stubtree_items |= set(
i.id
for i in item.targets
if hasattr(i, "id")
and (not i.id.startswith("_") or i.id.endswith("__"))
)
for target in item.targets:
if isinstance(target, ast.Tuple):
stubtree_items |= set(i.id for i in target.elts)
elif isinstance(item, ast.AnnAssign):
stubtree_items |= {item.target.id}
try:
all_ = ast.literal_eval(ast.unparse(get_subtree(tree, "__all__").value))
except ValueError:
all_ = []
if all_:
missing = (tree_items - stubtree_items) & set(all_)
else:
missing = tree_items - stubtree_items
deprecated = set()
for item_name in missing:
item = get_subtree(tree, item_name)
if hasattr(item, "decorator_list"):
if "deprecated" in [
i.func.attr
for i in item.decorator_list
if hasattr(i, "func") and hasattr(i.func, "attr")
]:
deprecated |= {item_name}
if missing - deprecated and ~ignore & MISSING_STUB:
print(f"{path}: {missing - deprecated} missing from stubs")
ret |= MISSING_STUB
count += 1
non_class_or_func = set()
for item_name in stubtree_items - tree_items:
try:
get_subtree(tree, item_name)
except ValueError:
pass
else:
non_class_or_func |= {item_name}
missing_implementation = stubtree_items - tree_items - non_class_or_func
if missing_implementation and ~ignore & MISSING_IMPL:
print(f"{path}: {missing_implementation} in stubs and not source")
ret |= MISSING_IMPL
count += 1
for item_name in tree_items & stubtree_items:
item = get_subtree(tree, item_name)
stubitem = get_subtree(stubtree, item_name)
if isinstance(item, ast.FunctionDef) and isinstance(stubitem, ast.FunctionDef):
err, c = check_function(item, stubitem, f"{path}::{item_name}", ignore)
ret |= err
count += c
if isinstance(item, ast.ClassDef):
# Ignore set differences for classes... while it would be nice to have
# inheritance and attributes set in init/methods make both presence and
# absence of nodes spurious
err, c = check_namespace(
item,
stubitem,
f"{path}::{item_name}",
ignore | MISSING_STUB | MISSING_IMPL,
)
ret |= err
count += c
return ret, count
def check_function(item, stubitem, path, ignore):
ret = 0
count = 0
# if the stub calls overload, assume it knows what its doing
overloaded = "overload" in [
i.id for i in stubitem.decorator_list if hasattr(i, "id")
]
if overloaded:
return 0, 0
item_posargs = [a.arg for a in item.args.posonlyargs]
stubitem_posargs = [a.arg for a in stubitem.args.posonlyargs]
if item_posargs != stubitem_posargs and ~ignore & POS_ARGS:
print(
f"{path} {item.name} posargs differ: {item_posargs} vs {stubitem_posargs}"
)
ret |= POS_ARGS
count += 1
item_args = [a.arg for a in item.args.args]
stubitem_args = [a.arg for a in stubitem.args.args]
if item_args != stubitem_args and ~ignore & ARGS:
print(f"{path} args differ for {item.name}: {item_args} vs {stubitem_args}")
ret |= ARGS
count += 1
item_vararg = item.args.vararg
stubitem_vararg = stubitem.args.vararg
if ~ignore & VARARG:
if (item_vararg is None) ^ (stubitem_vararg is None):
if item_vararg:
print(
f"{path} {item.name} vararg differ: "
f"{item_vararg.arg} vs {stubitem_vararg}"
)
else:
print(
f"{path} {item.name} vararg differ: "
f"{item_vararg} vs {stubitem_vararg.arg}"
)
ret |= VARARG
count += 1
elif item_vararg is None:
pass
elif item_vararg.arg != stubitem_vararg.arg:
print(
f"{path} {item.name} vararg differ: "
f"{item_vararg.arg} vs {stubitem_vararg.arg}"
)
ret |= VARARG
count += 1
item_kwonlyargs = [a.arg for a in item.args.kwonlyargs]
stubitem_kwonlyargs = [a.arg for a in stubitem.args.kwonlyargs]
if item_kwonlyargs != stubitem_kwonlyargs and ~ignore & KWARGS:
print(
f"{path} {item.name} kwonlyargs differ: "
f"{item_kwonlyargs} vs {stubitem_kwonlyargs}"
)
ret |= KWARGS
count += 1
item_kwarg = item.args.kwarg
stubitem_kwarg = stubitem.args.kwarg
if ~ignore & VARKWARG:
if (item_kwarg is None) ^ (stubitem_kwarg is None):
if item_kwarg:
print(
f"{path} {item.name} varkwarg differ: "
f"{item_kwarg.arg} vs {stubitem_kwarg}"
)
else:
print(
f"{path} {item.name} varkwarg differ: "
f"{item_kwarg} vs {stubitem_kwarg.arg}"
)
ret |= VARKWARG
count += 1
elif item_kwarg is None:
pass
elif item_kwarg.arg != stubitem_kwarg.arg:
print(
f"{path} {item.name} varkwarg differ: "
f"{item_kwarg.arg} vs {stubitem_kwarg.arg}"
)
ret |= VARKWARG
count += 1
return ret, count
def get_subtree(tree, name):
for item in tree.body:
if isinstance(item, ast.Assign):
if name in [i.id for i in item.targets if hasattr(i, "id")]:
return item
for target in item.targets:
if isinstance(target, ast.Tuple):
if name in [i.id for i in target.elts]:
return item
if isinstance(item, ast.AnnAssign):
if name == item.target.id:
return item
if not hasattr(item, "name"):
continue
if item.name == name:
return item
raise ValueError(f"no such item {name} in tree")
if __name__ == "__main__":
out = 0
count = 0
basedir = pathlib.Path("lib/matplotlib")
per_file_ignore = {
# Edge cases for items set via `get_attr`, etc
basedir / "__init__.py": MISSING_IMPL,
# Base class has **kwargs, subclasses have more specific
basedir / "ticker.py": VARKWARG,
basedir / "layout_engine.py": VARKWARG,
}
for f in basedir.rglob("**/*.py"):
err, c = check_file(f, ignore=0 | per_file_ignore.get(f, 0))
out |= err
count += c
print("\n")
print(f"{count} total errors found")
sys.exit(out)
You can’t perform that action at this time.