FilesExpand file tree

 master

/

cleaners.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

32 lines (28 loc) · 1.17 KB

 master

/

cleaners.py

Copy path

Top

File metadata and controls

• Code
• Blame

32 lines (28 loc) · 1.17 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

# strip out a set of nuisance html attributes that can mess up rendering in RSS feeds

import re

from lxml.html.clean import Cleaner

bad_attrs = ['style', '[-a-z]*color', 'background[-a-z]*', 'on*']

single_quoted = "'[^']+'"

double_quoted = '"[^"]+"'

non_space = '[^ "\'>]+'

htmlstrip = re.compile("<" # open

"([^>]+) " # prefix

"(?:%s) *" % ('|'.join(bad_attrs),) + # undesirable attributes

'= *(?:%s|%s|%s)' % (non_space, single_quoted, double_quoted) + # value

"([^>]*)" # postfix

">" # end

, re.I)

def clean_attributes(html):

while htmlstrip.search(html):

html = htmlstrip.sub('<\\1\\2>', html)

return html

def normalize_spaces(s):

if not s: return ''

"""replace any sequence of whitespace

characters with a single space"""

return ' '.join(s.split())

html_cleaner = Cleaner(scripts=True, javascript=True, comments=True,

style=True, links=True, meta=False, add_nofollow=False,

page_structure=False, processing_instructions=True, embedded=False,

frames=False, forms=False, annoying_tags=False, remove_tags=None,

remove_unknown_tags=False, safe_attrs_only=False)