共计 10221 个字符，预计需要花费 26 分钟才能阅读完成。
指标网站:https://myip.ms
难度 10 颗星
具备极强的反爬虫能力：封 ip
'''
------------------------------
https://myip.ms/browse/web_hosting/1/countryID/ALA%5EASM
------------------------------
'''
import os
import csv
import time
import random
import requests
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
class LoopOver(Exception):
def __init__(self, *args, **kwargs):
pass
class Spider:
def __init__(self):
# csv 贮存
self.path = '.'
self.inputfilename = 'country.csv'
self.csvfilename = 'datas.csv'
self.logfilename = 'run.log'
options = webdriver.ChromeOptions()
self.browser = webdriver.Chrome(options=options)
self.wait = WebDriverWait(self.browser, 20)
# 链接
self.listurl = 'https://myip.ms/browse/web_hosting/1/countryID/{}'
self.host = 'https://myip.ms'
self.tempalte = '''
<p>
{}
</p>
<table border="5">
<thead class="tableFloatingHeaderOriginal">
<tr valign="middle">
<th class="nobackgroundimage" align="center" style="width: 32px;">
No
</th>
<th colfirst="ip_owners" align="center" title-orig="Hosting Company" class="header" style="width: 163px;">
Hosting Company</th>
<th align="center" title-orig="Website/s" class="header" style="width: 114px;">
Website/s</th>
<th align="center" title-orig="Total Websites use this company IPs" class="header headerSortUp"
style="width: 92px;">
Total Websites use this company IPs</th>
<th align="center" title-orig="TOP Websites use this company IPs" class="header" style="width: 77px;">
TOP Websites use this company IPs</th>
<th align="center" title-orig="Diagram" class="header" style="width: 38px;">
Record Update Time</th>
</tr>
</thead>
<tbody>
{}
</tbody>
</table>
'''self.tempalte_page ='''
<table border="5">
<thead class="tableFloatingHeaderOriginal">
<tr valign="middle">
<th class="nobackgroundimage" align="center" style="width: 44px;"><div class="edit-icon-tmp normal ui-button ui-widget ui-state-default ui-corner-all ui-button-text-icon-primary" role="button" title-orig="View Table in Full-screen Mode" style="position: absolute; z-index: 1001; left: 5px; top: 568.6px; display: none;"><span class="ui-button-icon-primary ui-icon ui-icon-arrow-4-diag"></span><span class="ui-button-text">Full-screen Mode</span></div><div class="edit-icon-tmp normal ui-button ui-widget ui-state-default ui-corner-all ui-button-text-icon-primary" role="button" title-orig="View Table in Full-screen Mode" style="position: absolute; z-index: 1001; left: 5px; top: 568.6px; display: none;"><span class="ui-button-icon-primary ui-icon ui-icon-arrow-4-diag"></span><span class="ui-button-text">Full-screen Mode</span></div><div class="edit-icon-tmp normal ui-button ui-widget ui-state-default ui-corner-all ui-button-text-icon-primary" role="button" title-orig="View Table in Full-screen Mode" style="position: absolute; z-index: 1001; left: 5px; top: 568.6px; display: none;"><span class="ui-button-icon-primary ui-icon ui-icon-arrow-4-diag"></span><span class="ui-button-text">Full-screen Mode</span></div><div class="edit-icon-tmp normal ui-button ui-widget ui-state-default ui-corner-all ui-button-text-icon-primary" role="button" title-orig="View Table in Full-screen Mode" style="position: absolute; z-index: 1001; left: 5px; top: 568.6px; display: none;"><span class="ui-button-icon-primary ui-icon ui-icon-arrow-4-diag"></span><span class="ui-button-text">Full-screen Mode</span></div><div class="edit-icon-tmp normal ui-button ui-widget ui-state-default ui-corner-all ui-button-text-icon-primary" role="button" title-orig="View Table in Full-screen Mode" style="position: absolute; z-index: 1001; left: 5px; top: 568.6px; display: none;"><span class="ui-button-icon-primary ui-icon ui-icon-arrow-4-diag"></span><span class="ui-button-text">Full-screen Mode</span></div>No</th>
<th colfirst="sites" align="center" title-orig="Web Site" class="header" style="width: 153px;">
Web Site</th>
<th align="center" title-orig="Website IP Address" class="header" style="width: 144px;">
Website IP Address</th>
<th align="center" title-orig="Web Hosting Company / IP Owner" class="header" style="width: 178px;">
Website IPV6 Address</th>
<th align="center" title-orig="Web Hosting / Server IP Location" class="header" style="width: 134px;">
World Site Popular</th>
<th align="center" title-orig="Web Hosting City" class="header" style="width: 105px;">
World Site Popular Rating</th>
<th align="center" title-orig="World Site Popular Rating" class="header headerSortDown" style="width: 86px;">
DNS Records</th>
<th align="center" title-orig="Diagram" class="header" style="width: 38px;">
Record Update Time</th>
</tr>
</thead>
<tbody>
{}
</tbody>
</table>
'''
def turn2filename(self, dst):
d = dst.replace("\\", "").replace("/","").replace(":", "").replace("*","").replace("?", "").replace("\"","").replace("<", "").replace(">","").replace("|", "")
return d
def run(self):
strat = time.time()
self.get_input()
# 71
for c, cid in self.datas[115:116]:
print('>>>', c, self.listurl.format(cid))
for item_index, item in enumerate(self.parse_list(self.get_list(self.listurl.format(cid)))):
if item[1] == '- No Records Found -':
item[0] = c
if c in ['British Indian Ocean Territory', 'Brunei', 'Bulgaria']:
self.save_data(item=item, filename=self.turn2filename(c) + '.csv')
else:
self.save_data(item=item, filename='data.csv')
time.sleep(0)
end = time.time()
self.runtime = end - strat
print('用时{}'.format(self.runtime))
end = time.time()
self.runtime = end - strat
def get_input(self):
with open(self.inputfilename, 'r', encoding='utf_8') as f:
reader = csv.reader(f)
self.datas = [i for i in list(reader) if i]
def mkurl(self, kw):
for i in range(0, 1):
yield self.listurl.format(kw, i * 10)
def get_list(self, url):
while True:
try:
self.browser.get(url)
try:
self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="sites_tbl"or @id ="web_hosting_tbl"]')))
except Exception:
if 'a Robot' in self.browser.find_element_by_xpath('/html/body/div[2]/div/div/div/center').text:
self.browser.find_element_by_xpath('//*[@id="captcha_submit"]').click()
time.sleep(1)
raise Exception
return self.browser
except Exception as error:
print('error >>>', error)
if self.browser.current_url != url:
self.browser.quit()
self.browser = webdriver.Chrome()
self.wait = WebDriverWait(self.browser, 20)
time.sleep(1)
pass
def parse_list(self, response):
html = etree.HTML(response.page_source)
def pop(attr): return attr[0].strip().replace('\n', '').replace(' ','') if attr else ''for tr in html.xpath('//*[@id="web_hosting_tbl"]/tbody/tr[not(contains(@class,"expand"))]'):
No = tr.xpath('./td[1]/text()')[0].strip()
Hosting_Company = pop(tr.xpath('./td[2]/a/text()'))
page_url = pop(tr.xpath('./td[2]/a/@href'))
country_name = pop(tr.xpath('./td[3]/a/text()'))
Website = pop(tr.xpath('./td[4]/a/text()'))
Total_Websites_use_this_company_IPs = pop(tr.xpath('./td[5]/a/text()'))
TOP_Websites_use_this_company_IPs = pop(tr.xpath('./td[6]/a/text()'))
record_update_time = pop(tr.xpath('./td[7]/text()'))
yield [country_name, No, Hosting_Company, Website, Total_Websites_use_this_company_IPs,
TOP_Websites_use_this_company_IPs, record_update_time, self.host + page_url]
def get_page(self, url):
while True:
try:
self.browser.get(url)
try:
self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="sites_tbl"or @id ="web_hosting_tbl"]')))
except Exception as error:
print('//*[@id="sites_tbl"or @id ="web_hosting_tbl"] error', error)
if 'a Robot' in self.browser.find_element_by_xpath('/html/body/div[2]/div/div/div/center').text:
self.browser.find_element_by_xpath('//*[@id="captcha_submit"]').click()
time.sleep(5)
raise Exception
return self.browser
except Exception as error:
print('error >>>', error)
if self.browser.current_url != url:
self.browser.quit()
self.browser = webdriver.Chrome()
self.wait = WebDriverWait(self.browser, 20)
time.sleep(100)
pass
def parse_page(self, response):
text = response.page_source
html = etree.HTML(text)
def pop(attr):
return attr[0].strip().replace('\n', '').replace(' ','') if attr else ''l = len(html.xpath('//*[@id="sites_tbl" or @id ="web_hosting_tbl"]/tbody/tr[not(contains(@class,"expand"))]'))
print('len is', l)
try:
for i in range(1, l + 1):
tr = html.xpath('//*[@id="sites_tbl"or @id ="web_hosting_tbl"]/tbody/tr[not(contains(@class,"expand"))][{}]'.format(i))[0]
tre = html.xpath('//*[@id="sites_tbl"or @id ="web_hosting_tbl"]/tbody/tr[contains(@class,"expand")][{}]'.format(i))[0]
No = pop(tr.xpath('./td[1]/text()'))
web_site = pop(tr.xpath('./td[2]/a/text()'))
web_site_ip_address = pop(tr.xpath('./td[3]/a/text()'))
# tre
web_site_ipv6_address = pop(
tre.xpath('./td[1]/div[@class="stitle"]/b[contains(text(),"IPv6")]/../following-sibling::*[1]//a/text()'))
# tre
website_popularity = pop(tre.xpath('./td[1]/div/span[@class="bold arial grey"]/text()'))
website_popularity_rating = pop(tr.xpath('./td[7]/span/text()'))
# tre
dns_records = '\n'.join(
[i for i in tre.xpath('./td[1]/div[@class="stitle"]/b[contains(text(),"DNS")]/../following-sibling::*[1]//a/text()')])
# tre
record_update_time = pop(
tre.xpath('./td[1]/div[@class="stitle"]/b[contains(text(),"Record Update Time")]/../following-sibling::div/text()'))
yield [No, web_site, web_site_ip_address, web_site_ipv6_address, website_popularity,
website_popularity_rating, dns_records, record_update_time]
except IndexError:
raise LoopOver
if l < 50:
with open('error.html', 'w', encoding='utf-8') as f:
f.write(text)
raise LoopOver
def save_data(self, filename=None, path=None, item=None):
if not filename:
filename = self.csvfilename
if not path:
path = self.path
'''保留文件'''
with open('{}/{}'.format(path, filename), 'a', encoding='utf_8', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(item)
def save_log(self, info):
with open(self.logfilename, 'a', encoding='utf-8') as f:
f.write(info + '' + time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime()) + '\n')
def save_html_list(self, country, items, filename=None, path=None):
tr = ''
for item in items:
t = ''
for index, it in enumerate(item):
if index == 1:
td = '<td><a href="./data/{}-{}.html">{}</a></td>'.format(country, it.replace("\\", "").replace("/","").replace(":", "").replace("*","").replace("?",
"").replace("\"","").replace("<", "").replace(">","").replace("|", ""), it)
else:
td = '<td>{}</td>'.format(it)
t += td
tr += '<tr>' + t + '</tr>'
with open('main.html', 'a', encoding='utf-8') as f:
f.write(self.tempalte.format(country, tr))
def save_html_page(self, country, items, filename=None, path=None, it=None):
if not os.path.exists(path):
os.mkdir(path)
tr = ''
for index, item in enumerate(items):
t = ''
for it in item:
td = '<td>{}</td>'.format(it)
t += td
tr += '<tr>' + t + '</tr>'
with open('./{}/{}'.format(path, filename), 'w', encoding='utf-8') as f:
f.write(self.tempalte_page.format(tr))
@property
def time(self):
return '总共用时：{}秒'.format(self.runtime)
if __name__ == '__main__':
spider = Spider()
spider.run()
print(spider.time) # 运行总工夫
正文完
