1+ import requests
2+ from bs4 import BeautifulSoup
3+ import os
4+ import json
5+ import re
6+
7+ class TaobaoMeizi :
8+
9+ def __init__ (self ):
10+ self .data = {'q' : '' ,
11+ 'viewFlag' : 'A' ,
12+ 'sortType' : 'default' ,
13+ 'searchStyle' : '' ,
14+ 'searchRegion' : 'city:' ,
15+ 'searchFansNum' : '' ,
16+ 'currentPage' : 1 ,
17+ 'pageSize' : 100 }
18+
19+ self .headers = {"user-agent" : "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36" }
20+ self .model_information_list = []
21+ self .session = requests .session ()
22+
23+ def get_everone_model_information (self ):
24+ url = "https://mm.taobao.com/tstar/search/tstar_model.do?_input_charset=utf-8"
25+ resp = self .session .post (url , headers = self .headers , data = self .data )
26+ # print(resp.cookies.get_dict())
27+ json_resp = json .loads (resp .text )
28+ print ("开始爬取第 {} 页的所有模特信息" .format (json_resp ['data' ]['currentPage' ]))
29+ for information in json_resp ['data' ]['searchDOList' ]:
30+ data_dict = {}
31+ data_dict ['realname' ] = information ['realName' ]
32+ data_dict ['userid' ] = information ['userId' ]
33+ height = information ['height' ]
34+ weight = information ['weight' ]
35+ city = information ['city' ]
36+ print ("模特: {} 身高: {} 体重: {} 城市: {} id: {}" .format (data_dict ['realname' ], height , weight , city , data_dict ['userid' ]))
37+ self .model_information_list .append (data_dict )
38+
39+ def get_model_all_photo (self ):
40+ flag = True
41+ while flag :
42+ data = self .model_information_list .pop ()
43+ print ("正在获取模特: {} 的所有相册......" .format (data ['realname' ]))
44+ # self.get_all_photo(data['userid'], data['realname'])
45+ userid = data ['userid' ]
46+ realname = data ['realname' ]
47+ if self .model_information_list == []:
48+ break
49+ flags = True
50+ page = 1
51+ while flags :
52+ url = "https://mm.taobao.com/self/album/open_album_list.htm?_charset=utf-8&user_id%20={}&page={}" .format (userid , page )
53+ photo_resp = requests .get (url , headers = self .headers ).text
54+ bsoup = BeautifulSoup (photo_resp , 'lxml' )
55+ bfind_title = bsoup .find_all ('h4' ) # 获取相册的标题
56+ bfind_first = bsoup .find_all ('a' , class_ = "mm-first" )
57+ bfind_pic_number = bsoup .find_all ('span' , class_ = "mm-pic-number" )
58+ re_album_id = re .compile (r'album_id=(.*?)&' ) # 匹配album id
59+ re_img_dict = re .compile (r'<input type="hidden" value="\[(.*?)\]' )
60+ re_picid = re .compile (r'"picId":"(.*?)"' ) # 获取每张图片的id
61+ get_album_id = re_album_id .findall (str (bfind_first )) # 得到album id
62+ title = [i .get_text ().strip () for i in bfind_title ] # 去除标题中的空格
63+ photo_number = [i .get_text ().strip ()[1 :- 2 ] for i in bfind_pic_number ] # 每个相册的图片总数
64+ if bfind_title == []:
65+ print ("模特 {} 爬完了~" .format (realname ))
66+ break
67+ for album_id , title , photo_num in zip (get_album_id , title , photo_number ):
68+ photo_url = "https://mm.taobao.com/photo-{}-{}.htm" .format (userid , album_id )
69+ resp = requests .get (photo_url ).text
70+ get_img = re_img_dict .search (resp )
71+ get_pic = re_picid .findall (get_img .groups ()[0 ])
72+ print ("正在下载 {} 的相册 {} 共计图片 {}..." .format (realname , title , photo_num ))
73+ for pic_id in get_pic :
74+ cookies = self .session .cookies .get_dict () # 没有cookie就没法得到json
75+ # 要想得到json~要提交post
76+ data = {
77+ 'album_user_id' : userid ,
78+ '_tb_token_' : cookies ['_tb_token_' ],
79+ 'pic_id' : pic_id ,
80+ 'album_id' : '' ,
81+ 'is_edit' : 'true'
82+ }
83+ get_photo_url = "https://mm.taobao.com/album/json/get_photo_data.htm?_input_charset=utf-8"
84+ get_photo_resp = self .session .post (get_photo_url , headers = self .headers , data = data , cookies = cookies ).text
85+ replace_str = (get_photo_resp .replace ('false' , '"false"' )).replace ('true' , '"true"' )
86+ try :
87+ img_url = (json .loads (replace_str )['photo_url' ]).replace ('_620x10000.jpg' , '' )[2 :]
88+ # print(img_url)
89+ self .download (realname , title , img_url )
90+ except json .JSONDecodeError :
91+ replace_strs = (get_photo_resp .replace ('false' , '"false"' )).replace ('true' , '"true"' )
92+ re_pic_url = re .compile (r'"picUrl":"//(.*?)",' )
93+ for img_url in re_pic_url .findall (replace_strs ):
94+ # print(img_url)
95+ self .download (realname , title , img_url )
96+ print ("相册 {} 爬取完成!" .format (title ))
97+ page += 1
98+
99+ def download (self , modelname , photo , img_url ):
100+ try :
101+ os .mkdir ('TaoBaoModel' )
102+ os .mkdir ("TaoBaoModel/{}" .format (modelname ))
103+ except FileExistsError :
104+ pass
105+ try :
106+ os .mkdir ("TaoBaoModel/{}/{}" .format (modelname , photo ))
107+ except FileExistsError :
108+ pass
109+ try :
110+ with open ("TaoBaoModel/{}/{}/{}.{}" .format (modelname , photo , (img_url .split ("/" )[4 ]).split ("_" )[0 ], img_url .split ('.' )[- 1 ]), 'wb' ) as d :
111+ resp = requests .get ("http://" + img_url , headers = self .headers , cookies = self .session .cookies .get_dict ())
112+ d .write (resp .content )
113+ except FileNotFoundError :
114+ pass
115+
116+ def main (self ):
117+ self .get_everone_model_information ()
118+ self .get_model_all_photo ()
119+
120+ if __name__ == '__main__' :
121+ taobaomeizi = TaobaoMeizi ()
122+ taobaomeizi .main ()
0 commit comments