Java基于jsoup实现的爬虫技术

package cn.tedu.service.impl; import cn.tedu.dao.JobDao; import cn.tedu.dao.impl.JobDaoImpl; import cn.tedu.entity.Job; import cn.tedu.entity.JobInfo; import cn.tedu.service.JobInfoService; import cn.tedu.service.JobService; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.io.File; import java.io.IOException; import java.util.List; public class JobServiceImpl implements JobService { JobDao jobDao = new JobDaoImpl(); JobInfoService jobInfoService = new JobInfoServiceImpl(); @Override public void save() { // 查询t_job表中的数据条数 int count = jobDao.count(); // 如果没有数据, 就爬取数据并保存 if (count <= 0) { List<JobInfo> jobInfos = jobInfoService.list(); for (JobInfo jobInfo : jobInfos) { // 网页地址 String url = "D:\\51job\\" + jobInfo.getName(); File file = new File(url); // 创建文档对象 Document doc = null; try { // 加载网页, 51job采用GBK编码 doc = Jsoup.parse(file, "GBK"); } catch (IOException e) { e.printStackTrace(); } // 获取class值为e的元素(DIV标签) Elements es = doc.getElementsByClass("e"); for (Element e : es) { // 岗位名 Elements elements1 = e.getElementsByClass("jname"); String name = ""; for (Element e1 : elements1) { name = e1.text(); } // 公司名 Elements elements2 = e.getElementsByClass("cname"); String company = ""; for (Element e2 : elements2) { company = e2.text(); } // 地点 经验 学历 Elements elements3 = e.getElementsByClass("d"); String address = ""; for (Element e3 : elements3) { address = e3.text().split(" | ")[0]; // 取到地点 } // 薪资 Elements elements4 = e.getElementsByClass("sal"); String salary = ""; for (Element e4 : elements4) { salary = e4.text(); } // 发布日期 Elements elements5 = e.getElementsByClass("time"); String pushDate = ""; for (Element e5 : elements5) { pushDate = e5.text(); } Job job = new Job(name, company, address, salary, pushDate); // 防止读取到空白数据 if (name != null && !name.equals("")) { jobDao.save(job); } } } } } @Override public List<Job> list() { return jobDao.list(); } }