FilesExpand file tree

 master

/

SpiderTopicData.py

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

158 lines (147 loc) · 4.94 KB

 master

/

SpiderTopicData.py

Copy path

Top

File metadata and controls

• Code
• Blame

158 lines (147 loc) · 4.94 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

# encoding: utf-8

"""

作者：liguoyu

"""

import time

import json

from Login import Login

from MyCode import config

import codecs

import random

class SpiderTopicData(object):

"""

获取知乎数据

"""

def __init__(self):

"""

初始化相关参数

"""

self.login = Login()

self.login.login()

self.headers = self.login.getHeaders()

self.data = self.login.getData()

self.session = self.login.getSession()

# self.session.encoding = 'utf8'

# print self.session.

# data link list

self.havefinished_list = [] # 已经处理的数据连接

self.waitting_list = [] # 未处理的数据连接

self.record_topic_data = dict()

self.record_topic_link_data = dict()

def setRootTopic(self,root_topic_id='19778317'):

self.waitting_list.append(root_topic_id)

def relogin(self):

"""

短线重连

:return:

"""

self.login.login()

self.headers = self.login.getHeaders()

self.data = self.login.getData()

self.session = self.login.getSession()

def getLinkTopic(self,link_url='https://www.zhihu.com/topic/19778317/organize/entire?parent=19778317'):

"""

获取连接下的数据

:param link_url:

:return: 数据 text

"""

# self.session

try:

res = self.session.post(link_url,data=self.data,headers=self.headers)

time.sleep(random.randint(4,12))

except:

self.relogin()

res = self.session.post(link_url,data=self.data,headers=self.headers)

time.sleep(random.randint(4,12))

topic = json.loads(res.text)

# topic = eval(res.text)

cur_topic = topic['msg'][0]

sub_topics = topic['msg'][1]

# for test

parent_topic_name = cur_topic[1].encode('utf8')

parent_topic_id = cur_topic[2]

sub_topics_name = []

sub_topics_id = []

for sub in sub_topics:

sub_topics_id.append(sub[0][2])

sub_topics_name.append(sub[0][1].encode('utf8'))

result = dict()

result["parent_topic_id"] = parent_topic_id

result["parent_topic_name"] = parent_topic_name

result["sub_topics_name"] = sub_topics_name

result["sub_topics_id"] = sub_topics_id

return result

def recordData(self,result):

"""

处理爬取的topic 数据，并记录

:param result:

:return:

"""

sub_topics_id = result['sub_topics_id']

sub_topics_name = result['sub_topics_name']

parent_topic_name = result['parent_topic_name']

parent_topic_id = result['parent_topic_id']

existed = False

child_topic_id = ''

if parent_topic_id not in self.record_topic_data:

self.record_topic_data[parent_topic_id] = parent_topic_name

for sub_id,sub_name in zip(sub_topics_id,sub_topics_name):

if sub_name == str("加载更多"):

existed = True

child_topic_id = sub_id

continue

if sub_id not in self.record_topic_data:

self.record_topic_data[sub_id] = sub_name

if sub_id not in self.record_topic_link_data:

self.record_topic_link_data[sub_id] = []

self.record_topic_link_data[sub_id].append(parent_topic_id)

return existed,parent_topic_id,child_topic_id

def getSubTopic(self,parent_topic_id='19778317',child_topic_id=''):

"""

爬取知乎某一话题下的所有子话题，仅爬取话题下一层（即仅爬取当前话题的孩子话题，孙子不管）

:param parent_topic_id: 当前话题 ID

:param child_topic_id: 子话题 ID应对显示不全时

:return:

"""

state = True

sub_topics_id = []

while state:

url_link = 'https://www.zhihu.com/topic/{0}/organize/entire'.format(parent_topic_id)

if child_topic_id:

url_link += "?child={}&parent={}".format(child_topic_id, parent_topic_id)

result = self.getLinkTopic(url_link)

sub_topics_id.extend(result['sub_topics_id'])

state,parent_topic_id,child_topic_id = self.recordData(result)

return sub_topics_id

def getAllTopic(self):

"""

爬取队列所有话题及子话题

:return:

"""

while self.waitting_list:

topic_id = self.waitting_list.pop(0)

if topic_id in self.havefinished_list:

continue

sub_topics = self.getSubTopic(parent_topic_id=topic_id)

self.waitting_list.extend((list(set(sub_topics))))

self.havefinished_list.append(topic_id)

print "当前一获取topic number:{}\t 当前已遍历 topic number: {}"\

.format(len(self.havefinished_list)+len(self.waitting_list),len(self.havefinished_list))

self.writeResulttoFile()

def writeResulttoFile(self,topic_file=config.TopicFilePath+"zhihu_topic.json",topic_link_file = config.TopicFilePath+'zhihu_topic_link.json'):

with codecs.open(topic_file,'w',encoding='utf8') as topic_fp:

json.dump(self.record_topic_data,topic_fp,ensure_ascii=False,encoding='utf8')

with codecs.open(topic_link_file,'w',encoding='utf8') as topic_link_fp:

json.dump(self.record_topic_link_data,topic_link_fp,ensure_ascii=False,encoding='utf8')

def main():

spiderData = SpiderTopicData()

spiderData.setRootTopic()

# result = spiderData.getLinkTopic()

# spiderData.recordData(result)

# spiderData.getSubTopic()

spiderData.getAllTopic()

# spiderData.writeResulttoFile()

# print set(spiderData.waitting_list)

print len(spiderData.havefinished_list)

if __name__ == "__main__":

main()