FilesExpand file tree

codeql-cli/v2.27.0

/

test_tokenizer.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

66 lines (55 loc) · 2.37 KB

codeql-cli/v2.27.0

/

test_tokenizer.py

Copy path

Top

File metadata and controls

• Code
• Blame

66 lines (55 loc) · 2.37 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

import sys

import os.path

import shutil

import unittest

import semmle.populator

from tests import test_utils

from semmle.python.parser import tokenizer

from blib2to3.pgen2.token import tok_name

def unescape(s):

return u"'" + s.replace(u"\\", u"\\\\").replace(u"\n", u"\\n").replace(u"\t", u"\\t").replace(u"\'", u"\\'") + u"'"

def format_token(token):

type, text, start, end = token

# Use Python 3 tokenize style output, regardless of version

token_range = u"%d,%d-%d,%d:" % (start + end)

return u"%-20s%-15s%s" % (token_range, tok_name[type], unescape(text))

class TokenizerTest(unittest.TestCase):

def __init__(self, name):

super(TokenizerTest, self).__init__(name)

self.test_folder = os.path.join(os.path.dirname(__file__), "tokenizer")

def setUp(self):

pass

def tearDown(self):

pass

def compare_tokens(self, filename):

pyfile = os.path.join(self.test_folder, filename)

tokenfile = os.path.join(self.test_folder, filename[:-3]+".tokens")

with open(tokenfile, "rb") as tkns:

expected = [ line.strip().decode("utf8") for line in tkns if line.strip() ]

try:

with open(pyfile, "rb") as srcfile:

srcbytes = srcfile.read()

encoding, srcbytes = tokenizer.encoding_from_source(srcbytes)

text = srcbytes.decode(encoding)

actual = [format_token(tkn) for tkn in tokenizer.Tokenizer(text).tokens()]

except Exception as ex:

print(ex)

self.fail("Failed to tokenize " + filename)

if expected == actual:

return

actualfile = os.path.join(self.test_folder, filename[:-3]+".actual")

with open(actualfile, "wb") as out:

for line in actual:

out.write(line.encode("utf8"))

out.write(b"\n")

lineno = 1

for expected_tkn, actual_tkn in zip(expected, actual):

assert type(expected_tkn) is str

assert type(actual_tkn) is str

self.assertEqual(expected_tkn, actual_tkn, " at %s:%d" % (filename[:-3]+".tokens", lineno))

lineno += 1

self.assertTrue(len(expected) == len(actual), "Too few or too many tokens for %s" % filename)

def test_tokens(self):

for file in os.listdir(self.test_folder):

if file.endswith(".py"):

self.compare_tokens(file)