FilesExpand file tree

 master

/

test_encoding.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

117 lines (95 loc) · 4.71 KB

 master

/

test_encoding.py

Copy path

Top

File metadata and controls

• Code
• Blame

117 lines (95 loc) · 4.71 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

from __future__ import absolute_import, division, unicode_literals

import os

import pytest

from .support import get_data_files, test_dir, errorMessage, TestData as _TestData

from html5lib import HTMLParser, _inputstream

def test_basic_prescan_length():

data = "<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')

pad = 1024 - len(data) + 1

data = data.replace(b"-a-", b"-" + (b"a" * pad) + b"-")

assert len(data) == 1024 # Sanity

stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)

assert 'utf-8' == stream.charEncoding[0].name

def test_parser_reparse():

data = "<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')

pad = 10240 - len(data) + 1

data = data.replace(b"-a-", b"-" + (b"a" * pad) + b"-")

assert len(data) == 10240 # Sanity

stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)

assert 'windows-1252' == stream.charEncoding[0].name

p = HTMLParser(namespaceHTMLElements=False)

doc = p.parse(data, useChardet=False)

assert 'utf-8' == p.documentEncoding

assert doc.find(".//title").text == "Caf\u00E9"

@pytest.mark.parametrize("expected,data,kwargs", [

("utf-16le", b"\xFF\xFE", {"override_encoding": "iso-8859-2"}),

("utf-16be", b"\xFE\xFF", {"override_encoding": "iso-8859-2"}),

("utf-8", b"\xEF\xBB\xBF", {"override_encoding": "iso-8859-2"}),

("iso-8859-2", b"", {"override_encoding": "iso-8859-2", "transport_encoding": "iso-8859-3"}),

("iso-8859-2", b"<meta charset=iso-8859-3>", {"transport_encoding": "iso-8859-2"}),

("iso-8859-2", b"<meta charset=iso-8859-2>", {"same_origin_parent_encoding": "iso-8859-3"}),

("iso-8859-2", b"", {"same_origin_parent_encoding": "iso-8859-2", "likely_encoding": "iso-8859-3"}),

("iso-8859-2", b"", {"same_origin_parent_encoding": "utf-16", "likely_encoding": "iso-8859-2"}),

("iso-8859-2", b"", {"same_origin_parent_encoding": "utf-16be", "likely_encoding": "iso-8859-2"}),

("iso-8859-2", b"", {"same_origin_parent_encoding": "utf-16le", "likely_encoding": "iso-8859-2"}),

("iso-8859-2", b"", {"likely_encoding": "iso-8859-2", "default_encoding": "iso-8859-3"}),

("iso-8859-2", b"", {"default_encoding": "iso-8859-2"}),

("windows-1252", b"", {"default_encoding": "totally-bogus-string"}),

("windows-1252", b"", {}),

])

def test_parser_args(expected, data, kwargs):

stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False, **kwargs)

assert expected == stream.charEncoding[0].name

p = HTMLParser()

p.parse(data, useChardet=False, **kwargs)

assert expected == p.documentEncoding

@pytest.mark.parametrize("kwargs", [

{"override_encoding": "iso-8859-2"},

{"override_encoding": None},

{"transport_encoding": "iso-8859-2"},

{"transport_encoding": None},

{"same_origin_parent_encoding": "iso-8859-2"},

{"same_origin_parent_encoding": None},

{"likely_encoding": "iso-8859-2"},

{"likely_encoding": None},

{"default_encoding": "iso-8859-2"},

{"default_encoding": None},

{"foo_encoding": "iso-8859-2"},

{"foo_encoding": None},

])

def test_parser_args_raises(kwargs):

with pytest.raises(TypeError) as exc_info:

p = HTMLParser()

p.parse("", useChardet=False, **kwargs)

assert exc_info.value.args[0].startswith("Cannot set an encoding with a unicode input")

def param_encoding():

for filename in get_data_files("encoding"):

tests = _TestData(filename, b"data", encoding=None)

for test in tests:

yield test[b'data'], test[b'encoding']

@pytest.mark.parametrize("data, encoding", param_encoding())

def test_parser_encoding(data, encoding):

p = HTMLParser()

assert p.documentEncoding is None

p.parse(data, useChardet=False)

encoding = encoding.lower().decode("ascii")

assert encoding == p.documentEncoding, errorMessage(data, encoding, p.documentEncoding)

@pytest.mark.parametrize("data, encoding", param_encoding())

def test_prescan_encoding(data, encoding):

stream = _inputstream.HTMLBinaryInputStream(data, useChardet=False)

encoding = encoding.lower().decode("ascii")

# Very crude way to ignore irrelevant tests

if len(data) > stream.numBytesMeta:

return

assert encoding == stream.charEncoding[0].name, errorMessage(data, encoding, stream.charEncoding[0].name)

# pylint:disable=wrong-import-position

try:

import chardet # noqa

except ImportError:

print("chardet not found, skipping chardet tests")

else:

def test_chardet():

with open(os.path.join(test_dir, "encoding", "chardet", "test_big5.txt"), "rb") as fp:

encoding = _inputstream.HTMLInputStream(fp.read()).charEncoding

assert encoding[0].name == "big5"

# pylint:enable=wrong-import-position