基于python的58同城招聘爬虫

from bs4 import BeautifulSoup import requests import time import pymongo import random client=pymongo.MongoClient('localhost',27017) xinzi=client['xinzi'] url_list=xinzi['zhaopin_url'] item_info=xinzi['item_info1'] proxy_list = [ 'http://111.23.10.44', 'http://115.231.128.81', 'http://111.23.10.29', 'http://111.8.22.209', 'http://111.23.10.123', 'http://171.8.79.91', 'http://111.23.10.37', 'http://111.8.22.216', 'http://61.176.215.34', ] proxy_ip = random.choice(proxy_list) # 随机获取代理ip proxies = {'http': proxy_ip} def get_links(diqu,pages): l=list(diqu) l[23]=str(pages) list_view=''.join(l) # print(list_view) wb_data=requests.get(list_view) time.sleep(1) soup=BeautifulSoup(wb_data.text,'lxml') ##jingzhun > dt > a if soup.find('a', 'next'): for link in soup.select('dt > a'): item_link=link.get('href') url_list.insert_one({'url': item_link}) print(item_link) else: get_link(list_view,pages) def get_link(list_view,page): wb_data = requests.get(list_view) wb_data = requests.get(list_view) time.sleep(1) soup = BeautifulSoup(wb_data.text, 'lxml') for link in soup.select('dt > a'): item_link = link.get('href') url_list.insert_one({'url': item_link}) print(item_link) #get_links('http://bj.58.com/job/pn1/?bd=1&param9047=1|2',20) #http://bj.58.com/zplvyoujiudian/30125423917748x.shtml?psid=133757456196169413658291718&entinfo=30125423917748_j&ytdzwdetaildj=0 def get_item_info(url): diqu1 = url.split('/')[2] diqu = diqu1.split('.')[0] diqus = {'xz': '徐州', 'nj': '南京', 'sh': '上海', 'bj': '北京'} wb_data = requests.get(url, proxies=proxies) soup = BeautifulSoup(wb_data.text, 'lxml') try: #body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_info > span.pos_title title=soup.select('div.pos_base_info > span.pos_title')[0].text #body > div.con > div.leftCon > div.item_con.pos_info > span #body > div.con > div.leftCon > div.item_con.pos_info > span title_info=soup.select('span.pos_name')[0].text #body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_info > span.pos_salary salary=soup.select('span.pos_salary')[0].text #body > div.con > div.leftCon > div.item_con.pos_info > div.pos_welfare > span:nth-child(1) fulis=soup.select('span.pos_welfare_item') daiyus=[] for fuli in fulis: daiyu=fuli.get_text() daiyus.append(daiyu) #body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_condition > span.item_condition.pad_left_none nums=soup.select('div.pos_base_condition > span')[0].text item_info.insert_one( {'title': title, 'diqu': diqus[diqu], 'title_info': title_info, 'salary': salary, 'daiyus': daiyus, 'nums':nums,'url': url}) print(title,diqus[diqu],title_info,salary,daiyus,nums) except AttributeError: pass except IndexError: pass except TypeError: pass # print(daiyus) #get_item_info('http://bj.58.com/zplvyoujiudian/30125423917748x.shtml?psid=133757456196169413658291718&entinfo=30125423917748_j&ytdzwdetaildj=0')