# coding: utf-8
"""
@Time : 11/7/2022 13:42
@Author: fff
@File: test2.py
@Software: PyCharm
"""
import asyncio,Class_function,random
from playwright.async_api import async_playwright
class spider:
def __init__(self):
self.Core_Function = Class_function.Class_function()
self.page_result_list=[]
self.request_list = []
self.response_list = []
self.list_url = []
self.HTML_list=[]
async def click_function(self,page):
'''
;模拟遍历点击
:return:
'''
try:
num222 = await page.evaluate('''
window.stop();
num222=document.querySelectorAll('*').length;
num222;
''')
# self.Core_Function.callback_logging.info(num222)
#print(num222)
await page.evaluate('''
window.scrollBy(1920, 50);
treeWalker = document.createTreeWalker(document);
num111=0
while (treeWalker.nextNode() && num111<1500) {
console.log("[*] processing node " + treeWalker.currentNode.tagName + ' ' + treeWalker.currentNode.id);
if (treeWalker.currentNode.click) {
treeWalker.currentNode.target='';
treeWalker.currentNode.click();
num111=num111+1;
}
}
''')
flag_num=0
while await page.evaluate('num111') < 666:
#self.Core_Function.callback_logging().info('while')
await asyncio.sleep(0.5)
flag_num=flag_num+1
if flag_num<3:
break
except Exception as e:
self.Core_Function.callback_logging().error(e)
async def hook_requset(self,route):
'''
# hook 请求包
:param route:
:return:
'''
# print(route.request.url)
if route.request.url == self.target_request['url']:
if self.target_request['body'] != 'Null':
await route.continue_(headers=self.target_request['headers'], method=self.target_request['method'],
post_data=self.target_request['body'])
elif self.target_request['method'] == 'GET':
await route.continue_(headers=self.target_request['headers'], method=self.target_request['method'])
else:
await route.continue_()
elif route.request.url != 'about:blank' and route.request.is_navigation_request():
# print(route.request.url)
if route.request.method == 'GET':
request = {"headers": route.request.headers, "method": route.request.method, "url": route.request.url,
"body": "Null"}
self.request_list.append(request)
elif self.target_request['body'] != 'Null':
request = {"headers": route.request.headers, "method": route.request.method,
"url": route.request.url, "body": route.request.post_data}
self.request_list.append(request)
# await route.continue_()
await route.abort(error_code='aborted')
else:
await route.continue_()
async def handle_popup(self, page):
'''
# 关闭click打开的新窗口
:param page:
:return:
'''
await page.close()
async def handle_dialog(self, dialog):
'''
# 处理alert之类
:param dialog:
:return:
'''
await dialog.dismiss()
async def handle_network_http_request(self, request):
'''
; 获取页面http请求
:param request:
:return:
'''
try:
if request.resource_type not in ['image', 'stylesheet', 'websocket', 'media', 'font']:
request_data = {}
# print(request.url)
if request.post_data == None:
request_data['body'] = 'Null'
else:
request_data['body'] = request.post_data
request_data['url'] = request.url
request_data['headers'] = request.headers
request_data['method'] = request.method
request_data['time'] = self.Core_Function.callback_time(0)
request_data['describe'] = 'Null'
request_data['status'] = 0
#print(request_data)
self.request_list.append(request_data)
except Exception as e:
self.Core_Function.callback_logging().error(e)
async def handle_http_response(self, response):
'''
# 处理http响应
'''
try:
# print(response.request.url)
response_data = {}
html_data={}
if response.request.url == self.target_request['url']:
if response.status in [200, 301, 302, 404, 500]:
response_data['body'] = self.target_request['body']
response_data['url'] = self.target_request['url']
response_data['headers'] = self.target_request['headers']
response_data['method'] = self.target_request['method']
response_data['http_status_code'] = response.status
response_data['headers_response'] = response.headers
html=await response.text()
html_md5=self.Core_Function.md5_convert(html)
html_data['html']=html
html_data['html_md5'] = html_md5
html_data['time'] = self.Core_Function.callback_time(0)
html_data['status']=0
response_data['html_md5']=html_md5
response_data['time'] = self.Core_Function.callback_time(0)
response_data['describe'] = 'Null'
response_data['status'] = 0
# print(response_data)
self.response_list.append(response_data)
self.HTML_list.append(html_data)
except Exception as e:
self.Core_Function.callback_logging().error(e)
async def page_data(self,page,request):
'''
;页面信息获取
:param page:
:param request:
:return:
'''
html=await page.content()
html_md5 = self.Core_Function.md5_convert(html)
html_data={}
html_data['html'] = html
html_data['html_md5'] = html_md5
html_data['time'] = self.Core_Function.callback_time(0)
html_data['status'] = 0
request['status'] = 0
request['html_md5'] = html_md5
request['title'] = await page.title()
print(request['title'])
await page.evaluate('''
list_href=[]
window.open = function(url) { console.log("new link: " + url);list_href.push(url); };
window.close = function () { return false; };
''')
await page.evaluate('''
list_href=[]
for(i=0;i<document.getElementsByTagName("a").length;i++){
list_href.push(document.getElementsByTagName("a")[i].href); //输出该页面的所有链接。
}
''')
# print(await page.content())
filename_img = "%s/%s.jpeg" % (self.Core_Function.create_image_path(), ''.join(
random.sample('ABCDEFGHIJKLMNOPQRSTUVWXYZ012345678zyxwvutsrqponmlkjihgfedcba', 15)))
try:
await page.screenshot(path=filename_img, type='jpeg', quality=15)
except Exception as error:
filename_img
