爬虫学习代码，包含beatifulSoup，scrapy框架....

from selenium import webdriver import time import json from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By class JiXinDa: def __init__(self): self.url = 'http://jxd.itheima.net/#/login' self.driver = webdriver.Chrome() def login_to_find(self): # 发送请求 self.driver.get(self.url) # 设置窗口最大化 self.driver.maximize_window() # 登录 self.driver.find_element_by_class_name('el-button').click() # 体验项目 self.driver.find_element_by_id('button').click() # 定位短信服务元素 sms_service_element = self.driver.find_element_by_xpath( '//div[contains(@class,"sider-bar") and contains(@class ,"scole-bar")]/ul/li[2]/div/div/span') webdriver.ActionChains(self.driver).move_to_element( sms_service_element).click(sms_service_element).perform() # 单击“服务日志” service_log = self.driver.find_element_by_xpath( '//div[contains(@class,"sider-bar") and contains(@class, "scole-bar")]/ul/li[2]/ul/li/ul/li[5]') webdriver.ActionChains(self.driver).move_to_element(service_log).click(service_log).perform() # 显示等待<input>标签加载 WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.XPATH, '//div[@class="ReceiveLog"]/div/form/div/div[1]//div/input'))) # 输入签名名称为“黑马头条” self.driver.find_element_by_xpath('//div[@class="ReceiveLog"]/div/form/div/div[1]//div/input').send_keys('黑马头条') # 应用名称，显示等待<input>标签加载 WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.XPATH, '//div[@class="ReceiveLog"]/div/form/div/div[4]//div/input'))) self.driver.find_element_by_xpath('//div[@class="ReceiveLog"]/div/form/div/div[4]//div/input').send_keys('黑马头条') # 单击“搜索”按钮 WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.XPATH, '//div[@class="el-row"]/div[5]//div/button[1]/span[1]'))) self.driver.find_element_by_xpath('//div[@class="el-row"]/div[5]//div/button[1]/span[1]').click() def get_data(self): # 提取网页表格的数据 title_li = [] for i in range(1, 9): title = self.driver.find_element_by_xpath( f'//div[@class="ReceiveLog"]/div[2]//thead//th[{i}]').text title_li.append(title) # 保存表头与表格行组合后的数据 content_info = [] # 获取每页表格行的数量 num = self.driver.find_elements_by_xpath( '//div[@class="ReceiveLog"]//table[@class="el-table__body"]//tr') for i in range(1, len(num) + 1): content = self.driver.find_element_by_xpath( f'//div[@class="ReceiveLog"]//' f'table[@class="el-table__body"]//tr[{i}]').text content_info.append(dict(zip(title_li, content.splitlines()))) return content_info def save_data(self, data): try: with open('jixinda.json', mode='a+', encoding='utf-8')as file: file.write(json.dumps(data, ensure_ascii=False)) except Exception as e: print(e) return False def run(self): self.login_to_find() num = 1 while True: button = self.driver.find_element_by_xpath( '//div[@class="ReceiveLog"]//div/button[@class="btn-next"]') # 判断“下一页”按钮是否可用，如果可用，则单击“下一页”按钮 if button.is_enabled(): data = self.get_data() self.save_data(data) print(f'正在保存第{num}页数据') print(data) num += 1 button.click() else: end_data = self.get_data() print(f'正在保存第{num}页数据') self.save_data(end_data) print(end_data) self.driver.close() break if __name__ == '__main__': jixinda = JiXinDa() jixinda.run()