FilesExpand file tree

 master

/

encoding.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

21 lines (20 loc) · 603 Bytes

 master

/

encoding.py

Copy path

Top

File metadata and controls

• Code
• Blame

21 lines (20 loc) · 603 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

import re

import chardet

def get_encoding(page):

text = re.sub('</?[^>]*>\s*', ' ', page)

enc = 'utf-8'

if not text.strip() or len(text) < 10:

return enc # can't guess

try:

diff = text.decode(enc, 'ignore').encode(enc)

sizes = len(diff), len(text)

if abs(len(text) - len(diff)) < max(sizes) * 0.01: # 99% of utf-8

return enc

except UnicodeDecodeError:

pass

res = chardet.detect(text)

enc = res['encoding']

#print '->', enc, "%.2f" % res['confidence']

if enc == 'MacCyrillic':

enc = 'cp1251'

return enc