FilesExpand file tree

 master

/

encoding.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

48 lines (43 loc) · 1.57 KB

 master

/

encoding.py

Copy path

Top

File metadata and controls

• Code
• Blame

48 lines (43 loc) · 1.57 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

import re

import chardet

def get_encoding(page):

# Regex for XML and HTML Meta charset declaration

charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)

pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)

xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')

declared_encodings = (charset_re.findall(page) +

pragma_re.findall(page) +

xml_re.findall(page))

# Try any declared encodings

if len(declared_encodings) > 0:

for declared_encoding in declared_encodings:

try:

page.decode(custom_decode(declared_encoding))

return custom_decode(declared_encoding)

except UnicodeDecodeError:

pass

# Fallback to chardet if declared encodings fail

text = re.sub('</?[^>]*>\s*', ' ', page)

enc = 'utf-8'

if not text.strip() or len(text) < 10:

return enc # can't guess

res = chardet.detect(text)

enc = res['encoding'] or 'utf-8'

#print '->', enc, "%.2f" % res['confidence']

enc = custom_decode(enc)

return enc

def custom_decode(encoding):

"""Overrides encoding when charset declaration

or charset determination is a subset of a larger

charset. Created because of issues with Chinese websites"""

encoding = encoding.lower()

alternates = {

'big5': 'big5hkscs',

'gb2312': 'gb18030',

'ascii': 'utf-8',

'MacCyrillic': 'cp1251',

}

if encoding in alternates:

return alternates[encoding]

else:

return encoding