FilesExpand file tree

 master

/

example04.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

31 lines (25 loc) · 837 Bytes

 master

/

example04.py

Copy path

Top

File metadata and controls

• Code
• Blame

31 lines (25 loc) · 837 Bytes

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

from urllib.parse import urljoin

import re

import requests

from bs4 import BeautifulSoup

def main():

headers = {'user-agent': 'Baiduspider'}

proxies = {

'http': 'http://122.114.31.177:808'

}

base_url = 'https://www.zhihu.com/'

seed_url = urljoin(base_url, 'explore')

resp = requests.get(seed_url,

headers=headers,

proxies=proxies)

soup = BeautifulSoup(resp.text, 'lxml')

href_regex = re.compile(r'^/question')

link_set = set()

for a_tag in soup.find_all('a', {'href': href_regex}):

if 'href' in a_tag.attrs:

href = a_tag.attrs['href']

full_url = urljoin(base_url, href)

link_set.add(full_url)

print('Total %d question pages found.' % len(link_set))

if __name__ == '__main__':

main()