Skip to content
Navigation Menu
{{ message }}
forked from DonJayamanne/pythonVSCode
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcodeanalyze.py
More file actions
362 lines (294 loc) · 10.8 KB
/
Copy pathcodeanalyze.py
File metadata and controls
362 lines (294 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
import bisect
import re
import token
import tokenize
class ChangeCollector(object):
def __init__(self, text):
self.text = text
self.changes = []
def add_change(self, start, end, new_text=None):
if new_text is None:
new_text = self.text[start:end]
self.changes.append((start, end, new_text))
def get_changed(self):
if not self.changes:
return None
self.changes.sort(key=lambda x: x[:2])
pieces = []
last_changed = 0
for change in self.changes:
start, end, text = change
pieces.append(self.text[last_changed:start] + text)
last_changed = end
if last_changed < len(self.text):
pieces.append(self.text[last_changed:])
result = ''.join(pieces)
if result != self.text:
return result
class SourceLinesAdapter(object):
"""Adapts source to Lines interface
Note: The creation of this class is expensive.
"""
def __init__(self, source_code):
self.code = source_code
self.starts = None
self._initialize_line_starts()
def _initialize_line_starts(self):
self.starts = []
self.starts.append(0)
try:
i = 0
while True:
i = self.code.index('\n', i) + 1
self.starts.append(i)
except ValueError:
pass
self.starts.append(len(self.code) + 1)
def get_line(self, lineno):
return self.code[self.starts[lineno - 1]:
self.starts[lineno] - 1]
def length(self):
return len(self.starts) - 1
def get_line_number(self, offset):
return bisect.bisect(self.starts, offset)
def get_line_start(self, lineno):
return self.starts[lineno - 1]
def get_line_end(self, lineno):
return self.starts[lineno] - 1
class ArrayLinesAdapter(object):
def __init__(self, lines):
self.lines = lines
def get_line(self, line_number):
return self.lines[line_number - 1]
def length(self):
return len(self.lines)
class LinesToReadline(object):
def __init__(self, lines, start):
self.lines = lines
self.current = start
def readline(self):
if self.current <= self.lines.length():
self.current += 1
return self.lines.get_line(self.current - 1) + '\n'
return ''
def __call__(self):
return self.readline()
class _CustomGenerator(object):
def __init__(self, lines):
self.lines = lines
self.in_string = ''
self.open_count = 0
self.continuation = False
def __call__(self):
size = self.lines.length()
result = []
i = 1
while i <= size:
while i <= size and not self.lines.get_line(i).strip():
i += 1
if i <= size:
start = i
while True:
line = self.lines.get_line(i)
self._analyze_line(line)
if not (self.continuation or self.open_count or
self.in_string) or i == size:
break
i += 1
result.append((start, i))
i += 1
return result
# Matches all backslashes before the token, to detect escaped quotes
_main_tokens = re.compile(r'(\\*)((\'\'\'|"""|\'|")|#|\[|\]|\{|\}|\(|\))')
def _analyze_line(self, line):
token = None
for match in self._main_tokens.finditer(line):
prefix = match.group(1)
token = match.group(2)
# Skip any tokens which are escaped
if len(prefix) % 2 == 1:
continue
if token in ["'''", '"""', "'", '"']:
if not self.in_string:
self.in_string = token
elif self.in_string == token:
self.in_string = ''
if self.in_string:
continue
if token == '#':
break
if token in '([{':
self.open_count += 1
elif token in ')]}':
self.open_count -= 1
if line and token != '#' and line.endswith('\\'):
self.continuation = True
else:
self.continuation = False
def custom_generator(lines):
return _CustomGenerator(lines)()
class LogicalLineFinder(object):
def __init__(self, lines):
self.lines = lines
def logical_line_in(self, line_number):
indents = count_line_indents(self.lines.get_line(line_number))
tries = 0
while True:
block_start = get_block_start(self.lines, line_number, indents)
try:
return self._block_logical_line(block_start, line_number)
except IndentationError as e:
tries += 1
if tries == 5:
raise e
lineno = e.lineno + block_start - 1
indents = count_line_indents(self.lines.get_line(lineno))
def generate_starts(self, start_line=1, end_line=None):
for start, end in self.generate_regions(start_line, end_line):
yield start
def generate_regions(self, start_line=1, end_line=None):
# XXX: `block_start` should be at a better position!
block_start = 1
readline = LinesToReadline(self.lines, block_start)
try:
for start, end in self._logical_lines(readline):
real_start = start + block_start - 1
real_start = self._first_non_blank(real_start)
if end_line is not None and real_start >= end_line:
break
real_end = end + block_start - 1
if real_start >= start_line:
yield (real_start, real_end)
except tokenize.TokenError:
pass
def _block_logical_line(self, block_start, line_number):
readline = LinesToReadline(self.lines, block_start)
shifted = line_number - block_start + 1
region = self._calculate_logical(readline, shifted)
start = self._first_non_blank(region[0] + block_start - 1)
if region[1] is None:
end = self.lines.length()
else:
end = region[1] + block_start - 1
return start, end
def _calculate_logical(self, readline, line_number):
last_end = 1
try:
for start, end in self._logical_lines(readline):
if line_number <= end:
return (start, end)
last_end = end + 1
except tokenize.TokenError as e:
current = e.args[1][0]
return (last_end, max(last_end, current - 1))
return (last_end, None)
def _logical_lines(self, readline):
last_end = 1
for current_token in tokenize.generate_tokens(readline):
current = current_token[2][0]
if current_token[0] == token.NEWLINE:
yield (last_end, current)
last_end = current + 1
def _first_non_blank(self, line_number):
current = line_number
while current < self.lines.length():
line = self.lines.get_line(current).strip()
if line and not line.startswith('#'):
return current
current += 1
return current
def tokenizer_generator(lines):
return LogicalLineFinder(lines).generate_regions()
class CachingLogicalLineFinder(object):
def __init__(self, lines, generate=custom_generator):
self.lines = lines
self._generate = generate
_starts = None
@property
def starts(self):
if self._starts is None:
self._init_logicals()
return self._starts
_ends = None
@property
def ends(self):
if self._ends is None:
self._init_logicals()
return self._ends
def _init_logicals(self):
"""Should initialize _starts and _ends attributes"""
size = self.lines.length() + 1
self._starts = [None] * size
self._ends = [None] * size
for start, end in self._generate(self.lines):
self._starts[start] = True
self._ends[end] = True
def logical_line_in(self, line_number):
start = line_number
while start > 0 and not self.starts[start]:
start -= 1
if start == 0:
try:
start = self.starts.index(True, line_number)
except ValueError:
return (line_number, line_number)
return (start, self.ends.index(True, start))
def generate_starts(self, start_line=1, end_line=None):
if end_line is None:
end_line = self.lines.length()
for index in range(start_line, end_line):
if self.starts[index]:
yield index
def get_block_start(lines, lineno, maximum_indents=80):
"""Approximate block start"""
pattern = get_block_start_patterns()
for i in range(lineno, 0, -1):
match = pattern.search(lines.get_line(i))
if match is not None and \
count_line_indents(lines.get_line(i)) <= maximum_indents:
striped = match.string.lstrip()
# Maybe we're in a list comprehension or generator expression
if i > 1 and striped.startswith('if') or striped.startswith('for'):
bracs = 0
for j in range(i, min(i + 5, lines.length() + 1)):
for c in lines.get_line(j):
if c == '#':
break
if c in '[(':
bracs += 1
if c in ')]':
bracs -= 1
if bracs < 0:
break
if bracs < 0:
break
if bracs < 0:
continue
return i
return 1
_block_start_pattern = None
def get_block_start_patterns():
global _block_start_pattern
if not _block_start_pattern:
pattern = '^\\s*(((def|class|if|elif|except|for|while|with)\\s)|'\
'((try|else|finally|except)\\s*:))'
_block_start_pattern = re.compile(pattern, re.M)
return _block_start_pattern
def count_line_indents(line):
indents = 0
for char in line:
if char == ' ':
indents += 1
elif char == '\t':
indents += 8
else:
return indents
return 0
def get_string_pattern():
start = r'(\b[uU]?[rR]?)?'
longstr = r'%s"""(\\.|"(?!"")|\\\n|[^"\\])*"""' % start
shortstr = r'%s"(\\.|\\\n|[^"\\])*"' % start
return '|'.join([longstr, longstr.replace('"', "'"),
shortstr, shortstr.replace('"', "'")])
def get_comment_pattern():
return r'#[^\n]*'
You can’t perform that action at this time.