from bs4 import BeautifulSoup
import requests
import time
import pymongo
import random
client=pymongo.MongoClient('localhost',27017)
xinzi=client['xinzi']
url_list=xinzi['zhaopin_url']
item_info=xinzi['item_info1']
proxy_list = [
'http://111.23.10.44',
'http://115.231.128.81',
'http://111.23.10.29',
'http://111.8.22.209',
'http://111.23.10.123',
'http://171.8.79.91',
'http://111.23.10.37',
'http://111.8.22.216',
'http://61.176.215.34',
]
proxy_ip = random.choice(proxy_list) # 随机获取代理ip
proxies = {'http': proxy_ip}
def get_links(diqu,pages):
l=list(diqu)
l[23]=str(pages)
list_view=''.join(l)
# print(list_view)
wb_data=requests.get(list_view)
time.sleep(1)
soup=BeautifulSoup(wb_data.text,'lxml')
##jingzhun > dt > a
if soup.find('a', 'next'):
for link in soup.select('dt > a'):
item_link=link.get('href')
url_list.insert_one({'url': item_link})
print(item_link)
else:
get_link(list_view,pages)
def get_link(list_view,page):
wb_data = requests.get(list_view)
wb_data = requests.get(list_view)
time.sleep(1)
soup = BeautifulSoup(wb_data.text, 'lxml')
for link in soup.select('dt > a'):
item_link = link.get('href')
url_list.insert_one({'url': item_link})
print(item_link)
#get_links('http://bj.58.com/job/pn1/?bd=1¶m9047=1|2',20)
#http://bj.58.com/zplvyoujiudian/30125423917748x.shtml?psid=133757456196169413658291718&entinfo=30125423917748_j&ytdzwdetaildj=0
def get_item_info(url):
diqu1 = url.split('/')[2]
diqu = diqu1.split('.')[0]
diqus = {'xz': '徐州', 'nj': '南京', 'sh': '上海', 'bj': '北京'}
wb_data = requests.get(url, proxies=proxies)
soup = BeautifulSoup(wb_data.text, 'lxml')
try:
#body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_info > span.pos_title
title=soup.select('div.pos_base_info > span.pos_title')[0].text
#body > div.con > div.leftCon > div.item_con.pos_info > span
#body > div.con > div.leftCon > div.item_con.pos_info > span
title_info=soup.select('span.pos_name')[0].text
#body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_info > span.pos_salary
salary=soup.select('span.pos_salary')[0].text
#body > div.con > div.leftCon > div.item_con.pos_info > div.pos_welfare > span:nth-child(1)
fulis=soup.select('span.pos_welfare_item')
daiyus=[]
for fuli in fulis:
daiyu=fuli.get_text()
daiyus.append(daiyu)
#body > div.con > div.leftCon > div.item_con.pos_info > div.pos_base_condition > span.item_condition.pad_left_none
nums=soup.select('div.pos_base_condition > span')[0].text
item_info.insert_one(
{'title': title, 'diqu': diqus[diqu], 'title_info': title_info, 'salary': salary, 'daiyus': daiyus, 'nums':nums,'url': url})
print(title,diqus[diqu],title_info,salary,daiyus,nums)
except AttributeError:
pass
except IndexError:
pass
except TypeError:
pass
# print(daiyus)
#get_item_info('http://bj.58.com/zplvyoujiudian/30125423917748x.shtml?psid=133757456196169413658291718&entinfo=30125423917748_j&ytdzwdetaildj=0')
gongzi.zip （12个子文件） 
gongzi 
main.py 583B xinzi_shouji.py 3KB __pycache__ xinzi_url.cpython-35.pyc 513B xinzi_shouji.cpython-35.pyc 3KB xinzi_url.py 408B counts.py 191B .idea misc.xml 229B mongoSettings.xml 576B modules.xml 264B workspace.xml 44KB inspectionProfiles profiles_settings.xml 228B gongzi.iml 398B
评论1