FilesExpand file tree

 master

/

tokenutil.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

110 lines (86 loc) · 2.99 KB

 master

/

tokenutil.py

Copy path

Top

File metadata and controls

• Code
• Blame

110 lines (86 loc) · 2.99 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

"""Token-related utilities"""

# Copyright (c) IPython Development Team.

# Distributed under the terms of the Modified BSD License.

from __future__ import absolute_import, print_function

from collections import namedtuple

from io import StringIO

from keyword import iskeyword

from . import tokenize2

from .py3compat import cast_unicode_py2

Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])

def generate_tokens(readline):

"""wrap generate_tokens to catch EOF errors"""

try:

for token in tokenize2.generate_tokens(readline):

yield token

except tokenize2.TokenError:

# catch EOF error

return

def line_at_cursor(cell, cursor_pos=0):

"""Return the line in a cell at a given cursor position

Used for calling line-based APIs that don't support multi-line input, yet.

Parameters

----------

cell: text

multiline block of text

cursor_pos: integer

the cursor position

Returns

-------

(line, offset): (text, integer)

The line with the current cursor, and the character offset of the start of the line.

"""

offset = 0

lines = cell.splitlines(True)

for line in lines:

next_offset = offset + len(line)

if next_offset >= cursor_pos:

break

offset = next_offset

else:

line = ""

return (line, offset)

def token_at_cursor(cell, cursor_pos=0):

"""Get the token at a given cursor

Used for introspection.

Parameters

----------

cell : unicode

A block of Python code

cursor_pos : int

The location of the cursor in the block where the token should be found

"""

cell = cast_unicode_py2(cell)

names = []

tokens = []

offset = 0

for tup in generate_tokens(StringIO(cell).readline):

tok = Token(*tup)

# token, text, start, end, line = tup

start_col = tok.start[1]

end_col = tok.end[1]

# allow '|foo' to find 'foo' at the beginning of a line

boundary = cursor_pos + 1 if start_col == 0 else cursor_pos

if offset + start_col >= boundary:

# current token starts after the cursor,

# don't consume it

break

if tok.token == tokenize2.NAME and not iskeyword(tok.text):

if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':

names[-1] = "%s.%s" % (names[-1], tok.text)

else:

names.append(tok.text)

elif tok.token == tokenize2.OP:

if tok.text == '=' and names:

# don't inspect the lhs of an assignment

names.pop(-1)

if offset + end_col > cursor_pos:

# we found the cursor, stop reading

break

tokens.append(tok)

if tok.token == tokenize2.NEWLINE:

offset += len(tok.line)

if names:

return names[-1]

else:

return ''