FilesExpand file tree

 master

/

token.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

113 lines (99 loc) · 2.03 KB

 master

/

token.py

Copy path

Top

File metadata and controls

• Code
• Blame

113 lines (99 loc) · 2.03 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

from __future__ import absolute_import

from itertools import count

from token import *

from parso._compatibility import py_version

_counter = count(N_TOKENS)

# Never want to see this thing again.

del N_TOKENS

COMMENT = next(_counter)

tok_name[COMMENT] = 'COMMENT'

NL = next(_counter)

tok_name[NL] = 'NL'

# Sets the attributes that don't exist in these tok_name versions.

if py_version >= 30:

BACKQUOTE = next(_counter)

tok_name[BACKQUOTE] = 'BACKQUOTE'

else:

RARROW = next(_counter)

tok_name[RARROW] = 'RARROW'

ELLIPSIS = next(_counter)

tok_name[ELLIPSIS] = 'ELLIPSIS'

if py_version < 35:

ATEQUAL = next(_counter)

tok_name[ATEQUAL] = 'ATEQUAL'

ERROR_DEDENT = next(_counter)

tok_name[ERROR_DEDENT] = 'ERROR_DEDENT'

FSTRING_START = next(_counter)

tok_name[FSTRING_START] = 'FSTRING_START'

FSTRING_END = next(_counter)

tok_name[FSTRING_END] = 'FSTRING_END'

FSTRING_STRING = next(_counter)

tok_name[FSTRING_STRING] = 'FSTRING_STRING'

EXCLAMATION = next(_counter)

tok_name[EXCLAMATION] = 'EXCLAMATION'

# Map from operator to number (since tokenize doesn't do this)

opmap_raw = """\

( LPAR

) RPAR

[ LSQB

] RSQB

: COLON

, COMMA

; SEMI

+ PLUS

- MINUS

* STAR

/ SLASH

| VBAR

& AMPER

< LESS

> GREATER

= EQUAL

. DOT

% PERCENT

` BACKQUOTE

{ LBRACE

} RBRACE

@ AT

== EQEQUAL

!= NOTEQUAL

<> NOTEQUAL

<= LESSEQUAL

>= GREATEREQUAL

~ TILDE

^ CIRCUMFLEX

<< LEFTSHIFT

>> RIGHTSHIFT

** DOUBLESTAR

+= PLUSEQUAL

-= MINEQUAL

*= STAREQUAL

/= SLASHEQUAL

%= PERCENTEQUAL

&= AMPEREQUAL

|= VBAREQUAL

@= ATEQUAL

^= CIRCUMFLEXEQUAL

<<= LEFTSHIFTEQUAL

>>= RIGHTSHIFTEQUAL

**= DOUBLESTAREQUAL

// DOUBLESLASH

//= DOUBLESLASHEQUAL

-> RARROW

... ELLIPSIS

! EXCLAMATION

"""

opmap = {}

for line in opmap_raw.splitlines():

op, name = line.split()

opmap[op] = globals()[name]

def generate_token_id(string):

"""

Uses a token in the grammar (e.g. `'+'` or `'and'`returns the corresponding

ID for it. The strings are part of the grammar file.

"""

try:

return opmap[string]

except KeyError:

pass

return globals()[string]