实战小项目：爬取拉钩网并作小型数据分析

#!usr/bin/env python # -*-coding:utf-8 -*- __author__='WYY' __date__='2017.03.29' #实战小项目：爬取拉钩网并作小型数据分析 import requests import json import xlwt import time import random class Spider(): def __init__(self): self.keyword=raw_input(u'请输入职位：') #获取数据 def getData(self,url): user_agents=['Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533+ \(KHTML, like Gecko) Element Browser 5.0', 'IBM WebExplorer /v0.94', 'Galaxy/1.0 [en] (Mac OS X 10.5.6; U; en)', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', 'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14', 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) \Version/6.0 Mobile/10A5355d Safari/8536.25', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) \Chrome/28.0.1468.0 Safari/537.36', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; TheWorld)'] index=random.randint(0, 9) user_agent=user_agents[index] #cookie要自己抓 headers={'User_agent':user_agent, 'cookie':'your cookies'} html=requests.get(url,headers=headers) data=json.loads(html.text) return data #获取职位信息并存入列表 def getPosition(self,url): data=self.getData(url) position=data['content']['positionResult']['result'] po_list=[] if position is not None: for i in position: main=[] main.append(i['companyFullName']) main.append(i['financeStage']) main.append(i['positionName']) main.append(i['positionLables']) main.append(i['salary']) main.append(i['city']) main.append(i['education']) main.append(i['workYear']) main.append(i['jobNature']) main.append(i['createTime']) po_list.append(main) return po_list #获取数据,存入一个大的list def saveDetail(self): self.New=int(raw_input(u'请输入要爬取的页数:')) self.time=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) print u'\n本地时间：',self.time print u'\n开始采集数据...' container=[] for page in range(1,self.New+1): self.url='https://www.lagou.com/jobs/positionAjax.json?px=new&first=true&pn='+str(page)+'&kd='+str(self.keyword) po_list=self.getPosition(self.url) time.sleep(3) print u'第',page,u'项完毕' container=container+po_list return container #将数据存入excel def saveAll(self): book=xlwt.Workbook() sheet =book.add_sheet(str(self.keyword), cell_overwrite_ok=True) container=self.saveDetail() print u'\n采集完毕' print u'\n准备将数据存入表格...' heads=[u'公司全名', u'融资状况', u'工作名称', u'标签', u'薪酬', u'城市', u'学历要求',u'经验要求',u'工作类型',u'数据创建时间'] ii=0 for head in heads: sheet.write(0,ii,head) ii+=1 i=1 for list in container: j=0 for one in list: sheet.write(i, j, one) j+=1 i+=1 book.save(str(self.keyword)+'.xls') print u'\n录入成功！' spider=Spider() spider.saveAll()