FilesExpand file tree

 master

/

SpiderTopicQuestions.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

129 lines (108 loc) · 3.01 KB

 master

/

SpiderTopicQuestions.py

Copy path

Top

File metadata and controls

• Code
• Blame

129 lines (108 loc) · 3.01 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

# -*- coding:utf8 -*-

"""

作者：liguoyu

"""

import time

import urllib2

import random

import re

import os

from MyCode import config

import codecs

class SpiderTopicQuestions(object):

"""

爬取知乎某一话题下的数据

"""

def __init__(self):

"""

初始化相关参数

"""

self.getQuestions_number = 200

self.savefilename = None

def setSaveFile(self,file=config.TopicFilePath+"topic_questions_test.csv"):

self.savefilename = file

def setQuestions_number(self,number=200):

self.getQuestions_number = number

def getOnePageContent(self,url='https://www.zhihu.com/topic/19588911/questions?page=2'):

"""

获取网页内容

:param url:

:return:

"""

# 构建请求的 request

request = urllib2.Request(url)

# 获取请求的页面 HTML 代码

pageCode = None

try:

response = urllib2.urlopen(request)

# 将页面代码转成 UTF-8

pageCode = response.read().decode('utf-8')

except:

print "page not found!"

return pageCode

def paserContent(self,text):

"""

解析 html text 内容提取问题 ID 和 问题文本

:param text:

:return:

"""

pattern = re.compile(r'class="question_link" href="/question/(.*?)">(.*?)</a>', re.S)

items = re.findall(pattern, text)

questions = []

for item in items:

if '40470324' in item[0]:

continue

question_id = item[0]

sub_items = item[1].split(">")

length = len(sub_items)-1

question_text = sub_items[length]

questions.append([question_id,question_text.encode('utf8')])

return questions

def getTopicQuestions(self,topic_id='19651442'):

base_url = 'https://www.zhihu.com/topic/{}/questions'.format(topic_id)

print base_url

state = True

i = 1

questions = []

while state:

url = base_url + "?page={}".format(i)

print url

i += 1

text = self.getOnePageContent(url)

if not text:

state = False

continue

page_questions = self.paserContent(text)

if not page_questions:

state = False

continue

questions.extend(page_questions)

if len(questions) > self.getQuestions_number:

break

time.sleep(random.randint(0, 2))

for question in questions:

question.append(topic_id)

return questions

def show(self,questions):

for q in questions:

print "question ID: {} \t question: {}\t topic: {}".format(q[0],q[1],q[2])

def writeResultToFile(self,questions):

if not os.path.exists(self.savefilename):

with codecs.open(self.savefilename,'a',encoding='utf8') as fp:

fp.write(u'ID\tQuestion\ttopicID\n')

with codecs.open(self.savefilename,'a',encoding='utf8') as fp:

for q in questions:

# print q[0],q[1],q[2]

fp.write(u"{}\t{}\t{}\n".format(q[0],unicode(q[1], "utf8"),q[2]))

def Spider(self,topic_id):

questions = self.getTopicQuestions(topic_id)

self.writeResultToFile(questions)

print "cur topic {} finished spider !\nspider data {}".format(topic_id,len(questions))

def main():

stq = SpiderTopicQuestions()

stq.setSaveFile()

questions = stq.getTopicQuestions()

stq.show(questions)

stq.writeResultToFile(questions)

if __name__ == '__main__':

main()