playwright自动化 + 招聘数据

import os import time import random import pandas as pd import numpy as np from playwright.sync_api import Playwright, sync_playwright def name_file(name): ix = 0 while True: filename = f'{name}_{ix}.xlsx' if os.path.exists(filename): ix += 1 else: return filename def get_new_page_info(context, Locator): with context.expect_page() as new_page_info: Locator.click() new_page = new_page_info.value new_page.wait_for_load_state() new_page.set_default_timeout(1000) if '滑动' in new_page.title(): sliding_path(new_page) new_page.wait_for_load_state() position_name = new_page.locator('xpath=/html/body/section[3]/div[1]/div[1]/span[1]').text_content() try: job_company = new_page.locator('xpath=/html/body/main/aside/div[3]').text_content() except: job_company = '' job_request = new_page.locator('xpath=/html/body/section[3]/div[1]/div[2]').text_content() salary = new_page.locator('xpath=/html/body/section[3]/div[1]/div[1]/span[2]').text_content() position_label = new_page.locator('xpath=/html/body/section[4]/div/div[1]').text_content() content = new_page.locator('xpath=/html/body/main/content/section[2]').text_content() new_page.close() return [position_name, job_company, job_request, salary, position_label, content] def sliding_path(page): # 定义滑块和包含容器 slider_box = page.locator('xpath=//*[@id="nc_1_n1z"]').bounding_box() contain_box = page.locator('xpath=//*[@id="nc_1__scale_text"]/span').bounding_box() distance = contain_box['width'] page.mouse.move(x=int(slider_box['x']), y=slider_box['y'] + slider_box['height'] / 2) page.mouse.down() size = 1000 scale = 3 tolerance = distance * 0.2 # 超过 lst = np.linspace(0, distance + tolerance, size) + np.random.normal(size=size, scale=scale) ix = np.array([i ** 2 for i in range(1, int(size ** 0.5 + 1))]) - 1 move_list = lst[ix] for move in move_list: page.mouse.move(x=int(slider_box['x']) + move, y=slider_box['y'] + slider_box['height'] / 2, steps=3) size = 100 scale = 10 # 返回 lst = np.linspace(move_list[-1], 300, size) + np.random.normal(size=size, scale=scale) ix = np.array([i ** 2 for i in range(1, int(size ** 0.5 + 1))]) - 1 move_list = lst[ix] for move in move_list: page.mouse.move(x=int(slider_box['x']) + move, y=slider_box['y'] + slider_box['height'] / 2, steps=10) page.mouse.move(x=int(slider_box['x']) + 300, y=slider_box['y'] + slider_box['height'] / 2, steps=3) page.mouse.up() page.wait_for_load_state() def run(playwright: Playwright) -> None: browser = playwright.chromium.connect_over_cdp('http://localhost:6568') context = browser.contexts[0] page = context.pages[0] info_list = [] try: for i in range(30): Locators = page.locator('xpath=//*[@id="lp-search-job-box"]/div[3]/section[1]/div[1]/div/div/div[1]/div/a/div[1]/div') for Locator in Locators.all(): info = get_new_page_info(context, Locator) time.sleep(0.2) print(info) info_list.append(info) page.locator('xpath=//span[@aria-label="right"]').click() time.sleep(1) page.wait_for_load_state() except Exception as e: print(e) df = pd.DataFrame(info_list, columns=['position_name', 'job_company', 'job_request', 'salary', 'position_label', 'content']) df.to_excel(name_file('猎聘'), index=False) # 测试 # locator = page.locator('xpath=//*[@id="lp-search-job-box"]/div[3]/section[1]/div[1]/div/div/div[1]/div/a/div[1]/div').nth(1) # info = get_new_page_info(context, locator) # print(info) with sync_playwright() as playwright: run(playwright)