爬虫系统 Python实现 后端

from flask import Flask, request, jsonify from dao.news_dao import NewsDao from util import pack_result, spider from flask_cors import CORS app = Flask(__name__) cors = CORS(app) # 爬取接口 @app.route('/spider_news', methods=['POST']) def spider_news(): sources = ['网易新闻', '搜狐新闻', '网易音乐', '搜狐体育'] # 得到要爬取的网站编号 my_json = request.get_json() option = my_json.get('option') # 保存结果 result_list = [] # 爬取新闻的短标题和网址 title_list, url_list = spider.get_url(option) # 存储长标题，内容，日期 content_list = [] date_list = [] come_from_list = [] # 爬取新闻的长标题和内容和日期 for url in url_list: content, date = spider.get_content(url=url, option=option) content_list.append(content) date_list.append(date) come_from_list.append(sources[option-1]) # 将数据封装 for i in range(len(url_list)): data = pack_result.get_news(url_list[i], title_list[i], content_list[i], date_list[i], come_from_list[i]) result_list.append(data) # 存入数据库 my_news = NewsDao() for i in result_list: urls = my_news.get_all_urls() if i['url'] not in urls: my_news.add_news(i) # 返回给前端消息 result_data = pack_result.get_result('', '爬取成功', 1001) return jsonify(result_data) @app.route('/get_news', methods=['POST']) def get_news(): # 得到要爬取的网站 my_json = request.get_json() keywords = my_json.get('keywords') # 对关键字进行处理 keywords_list = keywords.split(' ') result_keywords_list = [] for my_keywords in keywords_list: if my_keywords != '': result_keywords_list.append(my_keywords) # 从数据库查询所有信息 my_news = NewsDao() url_list, short_title_list, long_title_list, content_list, date_list = my_news.get_all_news() # 数据封装 result_list = [] result_data = {} for i in range(len(url_list)): data = pack_result.get_news(url_list[i], short_title_list[i], long_title_list[i], content_list[i], date_list[i]) result_list.append(data) # 如果关键字为空返回全部数据 if len(result_keywords_list) == 0: print(len(result_list)) result_data = pack_result.get_result(result_list, '获取成功', '2002') return jsonify(result_data) else: new_result_list = [] for i in range(len(result_list)): for key in result_keywords_list: if key in result_list[i]['short_title']: new_result_list.append(result_list[i]) continue print(len(new_result_list)) result_data = pack_result.get_result(new_result_list, '获取成功', '2002') return jsonify(result_data) @app.route('/get_content', methods=['POST']) def get_content(): my_json = request.get_json() url = my_json.get('url') my_news = NewsDao() long_title, content = my_news.get_content(url) data = { 'long_title': long_title, 'content': content } return data if __name__ == '__main__': app.run(host='0.0.0.0', port=5000)