FilesExpand file tree

 master

/

lexer.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

82 lines (65 loc) · 2.4 KB

 master

/

lexer.py

Copy path

Top

File metadata and controls

• Code
• Blame

82 lines (65 loc) · 2.4 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

#

# Copyright (C) 2009-2020 the sqlparse authors and contributors

# <see AUTHORS file>

#

# This module is part of python-sqlparse and is released under

# the BSD License: https://opensource.org/licenses/BSD-3-Clause

"""SQL Lexer"""

# This code is based on the SqlLexer in pygments.

# http://pygments.org/

# It's separated from the rest of pygments to increase performance

# and to allow some customizations.

from io import TextIOBase

from sqlparse import tokens

from sqlparse.keywords import SQL_REGEX

from sqlparse.utils import consume

class Lexer:

"""Lexer

Empty class. Leaving for backwards-compatibility

"""

@staticmethod

def get_tokens(text, encoding=None):

"""

Return an iterable of (tokentype, value) pairs generated from

`text`. If `unfiltered` is set to `True`, the filtering mechanism

is bypassed even if filters are defined.

Also preprocess the text, i.e. expand tabs and strip it if

wanted and applies registered filters.

Split ``text`` into (tokentype, text) pairs.

``stack`` is the initial stack (default: ``['root']``)

"""

if isinstance(text, TextIOBase):

text = text.read()

if isinstance(text, str):

pass

elif isinstance(text, bytes):

if encoding:

text = text.decode(encoding)

else:

try:

text = text.decode('utf-8')

except UnicodeDecodeError:

text = text.decode('unicode-escape')

else:

raise TypeError("Expected text or file-like object, got {!r}".

format(type(text)))

iterable = enumerate(text)

for pos, char in iterable:

for rexmatch, action in SQL_REGEX:

m = rexmatch(text, pos)

if not m:

continue

elif isinstance(action, tokens._TokenType):

yield action, m.group()

elif callable(action):

yield action(m.group())

consume(iterable, m.end() - pos - 1)

break

else:

yield tokens.Error, char

def tokenize(sql, encoding=None):

"""Tokenize sql.

Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream

of ``(token type, value)`` items.

"""

return Lexer().get_tokens(sql, encoding)