FilesExpand file tree

 master

/

test_encoding.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

67 lines (49 loc) · 2.18 KB

 master

/

test_encoding.py

Copy path

Top

File metadata and controls

• Code
• Blame

67 lines (49 loc) · 2.18 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

from __future__ import absolute_import, division, unicode_literals

import os

import unittest

try:

unittest.TestCase.assertEqual

except AttributeError:

unittest.TestCase.assertEqual = unittest.TestCase.assertEquals

from .support import get_data_files, TestData, test_dir, errorMessage

from html5lib import HTMLParser, inputstream

class Html5EncodingTestCase(unittest.TestCase):

def test_codec_name_a(self):

self.assertEqual(inputstream.codecName("utf-8"), "utf-8")

def test_codec_name_b(self):

self.assertEqual(inputstream.codecName("utf8"), "utf-8")

def test_codec_name_c(self):

self.assertEqual(inputstream.codecName(" utf8 "), "utf-8")

def test_codec_name_d(self):

self.assertEqual(inputstream.codecName("ISO_8859--1"), "windows-1252")

def runParserEncodingTest(data, encoding):

p = HTMLParser()

assert p.documentEncoding is None

p.parse(data, useChardet=False)

encoding = encoding.lower().decode("ascii")

assert encoding == p.documentEncoding, errorMessage(data, encoding, p.documentEncoding)

def runPreScanEncodingTest(data, encoding):

stream = inputstream.HTMLBinaryInputStream(data, chardet=False)

encoding = encoding.lower().decode("ascii")

# Very crude way to ignore irrelevant tests

if len(data) > stream.numBytesMeta:

return

assert encoding == stream.charEncoding[0], errorMessage(data, encoding, stream.charEncoding[0])

def test_encoding():

for filename in get_data_files("encoding"):

tests = TestData(filename, b"data", encoding=None)

for idx, test in enumerate(tests):

yield (runParserEncodingTest, test[b'data'], test[b'encoding'])

yield (runPreScanEncodingTest, test[b'data'], test[b'encoding'])

try:

try:

import charade # flake8: noqa

except ImportError:

import chardet # flake8: noqa

except ImportError:

print("charade/chardet not found, skipping chardet tests")

else:

def test_chardet():

with open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt"), "rb") as fp:

encoding = inputstream.HTMLInputStream(fp.read()).charEncoding

assert encoding[0].lower() == "big5"