FilesExpand file tree

master

/

normalize.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

194 lines (173 loc) · 6.37 KB

master

/

normalize.py

Copy path

Top

File metadata and controls

• Code
• Blame

194 lines (173 loc) · 6.37 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

# -*- coding: utf-8 -*-

from html.parser import HTMLParser

from urllib.parse import quote, unquote

try:

from html.parser import HTMLParseError

except ImportError:

# HTMLParseError was removed in Python 3.5. It could never be

# thrown, so we define a placeholder instead.

class HTMLParseError(Exception):

pass

from html.entities import name2codepoint

import sys

import re

import html

# Normalization code, adapted from

# https://github.com/karlcow/markdown-testsuite/

significant_attrs = ["alt", "href", "src", "title"]

whitespace_re = re.compile(r"\s+")

class MyHTMLParser(HTMLParser):

def __init__(self):

HTMLParser.__init__(self)

self.convert_charrefs = False

self.last = "starttag"

self.in_pre = False

self.output = ""

self.last_tag = ""

def handle_data(self, data):

after_tag = self.last == "endtag" or self.last == "starttag"

after_block_tag = after_tag and self.is_block_tag(self.last_tag)

if after_tag and self.last_tag == "br":

data = data.lstrip('\n')

if not self.in_pre:

data = whitespace_re.sub(' ', data)

if after_block_tag and not self.in_pre:

if self.last == "starttag":

data = data.lstrip()

elif self.last == "endtag":

data = data.strip()

self.output += data

self.last = "data"

def handle_endtag(self, tag):

if tag == "pre":

self.in_pre = False

elif self.is_block_tag(tag):

self.output = self.output.rstrip()

self.output += "</" + tag + ">"

self.last_tag = tag

self.last = "endtag"

def handle_starttag(self, tag, attrs):

if tag == "pre":

self.in_pre = True

if self.is_block_tag(tag):

self.output = self.output.rstrip()

self.output += "<" + tag

# For now we don't strip out 'extra' attributes, because of

# raw HTML test cases.

# attrs = filter(lambda attr: attr[0] in significant_attrs, attrs)

if attrs:

attrs.sort()

for (k,v) in attrs:

self.output += " " + k

if v in ['href','src']:

self.output += ("=" + '"' +

quote(unquote(v), safe='/') + '"')

elif v != None:

self.output += ("=" + '"' + html.escape(v,quote=True) + '"')

self.output += ">"

self.last_tag = tag

self.last = "starttag"

def handle_startendtag(self, tag, attrs):

"""Ignore closing tag for self-closing """

self.handle_starttag(tag, attrs)

self.last_tag = tag

self.last = "endtag"

def handle_comment(self, data):

self.output += '<!--' + data + '-->'

self.last = "comment"

def handle_decl(self, data):

self.output += '<!' + data + '>'

self.last = "decl"

def unknown_decl(self, data):

self.output += '<!' + data + '>'

self.last = "decl"

def handle_pi(self,data):

self.output += '<?' + data + '>'

self.last = "pi"

def handle_entityref(self, name):

try:

c = chr(name2codepoint[name])

except KeyError:

c = None

self.output_char(c, '&' + name + ';')

self.last = "ref"

def handle_charref(self, name):

try:

if name.startswith("x"):

c = chr(int(name[1:], 16))

else:

c = chr(int(name))

except ValueError:

c = None

self.output_char(c, '&' + name + ';')

self.last = "ref"

# Helpers.

def output_char(self, c, fallback):

if c == '<':

self.output += "&lt;"

elif c == '>':

self.output += "&gt;"

elif c == '&':

self.output += "&amp;"

elif c == '"':

self.output += "&quot;"

elif c == None:

self.output += fallback

else:

self.output += c

def is_block_tag(self,tag):

return (tag in ['article', 'header', 'aside', 'hgroup', 'blockquote',

'hr', 'iframe', 'body', 'li', 'map', 'button', 'object', 'canvas',

'ol', 'caption', 'output', 'col', 'p', 'colgroup', 'pre', 'dd',

'progress', 'div', 'section', 'dl', 'table', 'td', 'dt',

'tbody', 'embed', 'textarea', 'fieldset', 'tfoot', 'figcaption',

'th', 'figure', 'thead', 'footer', 'tr', 'form', 'ul',

'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'video', 'script', 'style'])

def normalize_html(html):

r"""

Return normalized form of HTML which ignores insignificant output

differences:

Multiple inner whitespaces are collapsed to a single space (except

in pre tags):

>>> normalize_html("<p>a \t b</p>")

'<p>a b</p>'

>>> normalize_html("<p>a \t\nb</p>")

'<p>a b</p>'

* Whitespace surrounding block-level tags is removed.

>>> normalize_html("<p>a b</p>")

'<p>a b</p>'

>>> normalize_html(" <p>a b</p>")

'<p>a b</p>'

>>> normalize_html("<p>a b</p> ")

'<p>a b</p>'

>>> normalize_html("\n\t<p>\n\t\ta b\t\t</p>\n\t")

'<p>a b</p>'

>>> normalize_html("<i>a b</i> ")

'<i>a b</i> '

* Self-closing tags are converted to open tags.

>>> normalize_html("<br />")

'<br>'

* Attributes are sorted and lowercased.

>>> normalize_html('<a title="bar" HREF="foo">x</a>')

'<a href="foo" title="bar">x</a>'

* References are converted to unicode, except that '<', '>', '&', and

'"' are rendered using entities.

>>> normalize_html("&forall;&amp;&gt;&lt;&quot;")

'\u2200&amp;&gt;&lt;&quot;'

"""

html_chunk_re = re.compile(r"(\<!\[CDATA\[.*?\]\]\>|\<[^>]*\>|[^<]+)")

try:

parser = MyHTMLParser()

# We work around HTMLParser's limitations parsing CDATA

# by breaking the input into chunks and passing CDATA chunks

# through verbatim.

for chunk in re.finditer(html_chunk_re, html):

if chunk.group(0)[:8] == "<![CDATA":

parser.output += chunk.group(0)

else:

parser.feed(chunk.group(0))

parser.close()

return parser.output

except HTMLParseError as e:

sys.stderr.write("Normalization error: " + e.msg + "\n")

return html # on error, return unnormalized HTML