一、获取索引页。
增加好 headers，params 间接 get 申请即可。
二、在索引页中获取详情页的 url。
在浏览器中间接关上这个 url，认真看浏览器上方中 url，你会发现 url 会跳转，就是说他进行了重定向。这里咱们须要应用 fiddler 这个抓包工具对他进行抓包。
你会发现他返回的是一段 js 代码，代码拼接成的 url 就是咱们想要的真正的 real_url，这里咱们间接用正则表达式，进行匹配、拼接就 ok 了。
三、获取详情页。
既然获取到了详情页的实在 url，剩下的结构一下申请头 headers，间接申请即可，这里我不举荐加 cookie 参数到申请头里，因为加上之后拜访次数过多，他会封 cookie，让你输出验证码进行验证。
四、总结。
我看网上说他是封 ip，封 cookie 的，然而我爬了几百篇并没有发现这个状况，之后遇到再补充。附上代码：
import requests
from lxml import etree
import re
import urllib3
# 重定向、封 ip、封 cookie
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def get_index(start_url, page):
params = {
'query': '美女',
'_sug_type_': '','sut':'1828','lkt':'1,1597298703444,1597298703444','s_from':'input','_sug_':'y','type':'2','sst0':'1597298703546','page': page,'ie':'utf8','w':'01019900','dr':'1',
}
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Cookie': 'ABTEST=5|1597380273|v1; SNUID=59BBCE9AEAEF4626F956F592EA96FE70; IPLOC=CN3709; SUID=B3512470160DA00A000000005F3616B1; JSESSIONID=aaapSleD9Qep4fAnwHYox; SUV=0004E05A702451B35F3616B1A47C7070; SUID=B35124706A13A00A000000005F3616B1',
'Host': 'weixin.sogou.com',
'Referer': 'https://weixin.sogou.com/weixin?query=%E7%BE%8E%E5%A5%B3&_sug_type_=&sut=1828&lkt=1%2C1597298703444%2C1597298703444&s_from=input&_sug_=y&type=2&sst0=1597298703546&page=4&ie=utf8&w=01019900&dr=1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
}
response = requests.get(url=start_url, params=params, headers=headers, verify=False)
if response.status_code == 200:
html = etree.HTML(response.text)
urls = html.xpath('//ul[@class="news-list"]/li//h3/a/@href')
for url in urls:
url = 'https://weixin.sogou.com/' + url
print(url)
yield url
else:
print('getting index page fail')
def get_real_url(url):
headers = {
'Host': 'weixin.sogou.com',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Referer': 'https://weixin.sogou.com/weixin?type=2&s_from=input&query=%E7%BE%8E%E5%A5%B3&ie=utf8&_sug_=n&_sug_type_=',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'Cookie': 'SUID=D85024704A21A00A000000005EFDF58C; SUV=0035E9CD702450D85EFDF58D4E7EB415; weixinIndexVisited=1; LCLKINT=121; LSTMV=468%2C24; pgv_pvi=2192807936; ABTEST=0|1596950375|v1; IPLOC=CN3709; SNUID=C32054007175DCCBC54D7C1071164832; JSESSIONID=aaaMv1RDeodhj5yAwCYox; PHPSESSID=34n87c5popss5ckq1gcinpc9s2',
}
response = requests.get(url=url, headers=headers, verify=False)
pattern = re.compile('\+=.*?\'(.*?)\';', re.S)
url_list = re.findall(pattern, response.text)
real_url = ''
for i in url_list:
real_url += i
real_url.replace('@', '')
return real_url
def get_detail(url):
headers = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'cache-control': 'max-age=0',
'if-modified-since': 'Fri, 14 Aug 2020 12:33:50 +0800',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'cross-site',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36',
}
response = requests.get(url, headers=headers,verify=False)
html = etree.HTML(response.text)
if html.xpath('//*[@id="activity-name"]/text()'):
title = html.xpath('//*[@id="activity-name"]/text()')[0].strip()
else:
title = ''result = {'title': title,}
return result
if __name__ == '__main__':
start_url = 'https://weixin.sogou.com/weixin'
for page in range(1, 11):
urls = get_index(start_url=start_url, page=page)
for url in urls:
real_url = get_real_url(url)
result = get_detail(real_url)
print(result)