FilesExpand file tree

 main

/

test.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

25 lines (20 loc) · 598 Bytes

 main

/

test.py

Copy path

Top

File metadata and controls

• Code
• Blame

25 lines (20 loc) · 598 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

from . import test_tokenizer

import sys

from blib2to3.pgen2.token import tok_name

def printtoken(type, token, start, end): # for testing

token_range = "%d,%d-%d,%d:" % (start + end)

print("%-20s%-15s%r" %

(token_range, tok_name[type], token)

)

def main():

verbose = sys.argv[1] == "-v"

if verbose:

inputfile = sys.argv[2]

else:

inputfile = sys.argv[1]

with open(inputfile, "r") as input:

t = test_tokenizer.Tokenizer(input.read()+"\n")

for tkn in t.tokens(verbose):

printtoken(*tkn)

if __name__ == "__main__":

main()