Python Flask 疫情大数据可视化网站

#-*- coding = utf-8 -*- #@Time : 2020/8/7 17:12 #@Author : 刘 #@File : spider.py #@Software : PyCharm import sys import requests import json import time import pymysql import traceback from selenium.webdriver import Chrome,ChromeOptions def get_tenxun_Data(): url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_other' hear = { "user-agent": " Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" } res = requests.get(url, hear) d = json.loads(res.text) data_all_other = json.loads(d["data"]) history = {} # 存储历史记录 # {'confirm': 41, 'suspect': 0, 'dead': 1, 'heal': 0, 'nowConfirm': 0, # 'nowSevere': 0, 'importedCase': 0, 'deadRate': '2.4', 'healRate': '0.0', # 'date': '01.13', 'noInfect': 0} for i in data_all_other["chinaDayList"]: ds = "2021." + i["date"] tup = time.strptime(ds, "%Y.%m.%d") ds = time.strftime("%Y-%m-%d", tup) confirm = i["confirm"] # 确认 suspect = i["suspect"] # 怀疑 heal = i["heal"] # 治愈 dead = i["dead"] # 死 history[ds] = {"confirm": confirm, "suspect": suspect, "heal": heal, "dead": dead} for i in data_all_other["chinaDayAddList"]: ds = "2021." + i["date"] tup = time.strptime(ds, "%Y.%m.%d") ds = time.strftime("%Y-%m-%d", tup) confirm = i["confirm"] # 确认 suspect = i["suspect"] # 怀疑 heal = i["heal"] # 治愈 dead = i["dead"] # 死 history[ds].update({"confirm_add": confirm, "suspect_add": suspect, "heal_add": heal, "dead_add": dead}) # print(history) url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5' r = requests.get(url) rd = json.loads(r.text) data_all_h5 = json.loads(rd["data"]) # print(data_all.keys()) # dict_keys(['lastUpdateTime', 'chinaTotal', 'chinaAdd', # 'isShowAdd', 'showAddSwitch', 'areaTree']) details = [] # 单日详细数据 update_time = data_all_h5["lastUpdateTime"] data_country = data_all_h5["areaTree"] # 国家 data_province = data_country[0]["children"] # 中国各省份 # {'name': '香港', 'today': {'confirm': 246, 'confirmCuts': 0, # 'isUpdated': True}, 'total': {'nowConfirm': 1508, 'confirm': 3397, # 'suspect': 0, 'dead': 31, 'deadRate': '0.91', 'showRate': False, # 'heal': 1858, 'healRate': '54.70', 'showHeal': True}} for pro in data_province: province = pro["name"] # 各省份名字 for city_info in pro["children"]: city = city_info["name"] # 城市名字 confirm = city_info["total"]["confirm"] # 确认人数 confirm_add = city_info["today"]["confirm"] # 确认新增加 heal = city_info["total"]["heal"] # 治愈 dead = city_info["total"]["dead"] # 死 details.append([update_time, province, city, confirm, confirm_add, heal, dead]) # print(details) return history, details def get_conn(): conn = pymysql.connect(host="120.78.0.225",user="root",password="123456",db="cov",charset="utf8") cur = conn.cursor() return conn,cur def close_conn(conn,cur): if conn: conn.close() if cur: cur.close() def update_details(): ''' 更新details表 ''' cur = None con = None try: li = get_tenxun_Data()[1] #0是历史数据字典，1是最新数据 con,cur = get_conn() sql = ''' insert into details(update_time,province,city,confirm,confirm_add,heal,dead) values(%s,%s,%s,%s,%s,%s,%s) ''' sql_query = "select %s = (select update_time from details order by id desc limit 1 )" #对比当前最大时间 cur.execute(sql_query,li[0][0]) if not cur.fetchone()[0]: print(f"{time.asctime()}开始更新数据") for item in li: cur.execute(sql,item) con.commit() print(f"{time.asctime()}更新数据完毕") else: print(f"{time.asctime()}已经是最新数据！") except: traceback.print_exc() finally: close_conn(con,cur) def insert_history(): ''' insert_history 插入历史数据 ''' cur = None con = None try: dic = get_tenxun_Data()[0] # 0是历史数据字典，1是最新数据 print(f"{time.asctime()}开始插入数据") con, cur = get_conn() sql = ''' insert into history values(%s,%s,%s,%s,%s,%s,%s,%s,%s) ''' for k, v in dic.items(): # item格式 {"2020-08-01":{"confirm":41,"suspect"}:0} cur.execute(sql, [k, v.get("confirm"), v.get("confirm_add"), v.get("suspect"), v.get("suspect_add"), v.get("heal"), v.get("heal_add"), v.get("dead"), v.get("dead_add")]) con.commit() print(f"{time.asctime()}插入数据完毕") except: traceback.print_exc() finally: close_conn(con, cur) def update_history(): ''' update_history 更新历史数据 ''' cur = None con = None try: dic = get_tenxun_Data()[0] #0是历史数据字典，1是最新数据 print(f"{time.asctime()}开始更新数据") con,cur = get_conn() sql = ''' insert into history values(%s,%s,%s,%s,%s,%s,%s,%s,%s) ''' sql_query = "select confirm from history where ds=%s" for k,v in dic.items(): print(k) print(v) #item格式 {"2020-08-01":{"confirm":41,"suspect"}:0} if not cur.execute(sql_query,k): cur.execute(sql,[k,v.get("confirm"),v.get("confirm_add"),v.get("suspect"), v.get("suspect_add"),v.get("heal"),v.get("heal_add"), v.get("dead"),v.get("dead_add")]) con.commit() print(f"{time.asctime()}历史数据更新完毕") except: traceback.print_exc() finally: close_conn(con,cur) def get_baiduresou(): option = ChromeOptions() option.add_argument("--headless") # 隐藏浏览器 option.add_argument("--no-sandbox") data = [] url = 'http://top.baidu.com/buzz?b=1&fr=topindex' browser = Chrome(options=option) browser.get(url) c = browser.find_elements_by_xpath('//*[@id="main"]/div[2]/div/table/tbody/tr/td[2]/a[1]') context = [i.text for i in c] b = browser.find_elements_by_xpath('//*[@id="main"]/div[2]/div/table/tbody/tr/td[4]/span') num = [i.text for i in b] # print(context) return context, num def update_baidu(): cursor = None conn = None try: context, num = get_baiduresou() print(f"{time.asctime()}开始更新热搜数据") conn, cursor = get_conn() sql = ''' insert into hotsearch(dt,content,num) values(%s,%s,%s) ''' ts = time.strftime("%Y-%m-%d %X") for i, n in zip(context, num): cursor.execute(sql, (ts, i, n)) conn.commit() print(f"{time.asctime()}数据更新完毕") except: traceback.print_exc() finally: close_conn(conn, cursor) if __name__ == '__main__': l = len(sys.argv) if l == 1: s = ''' 请输入参数： 参数说明： update_history（up_his） 更新历史记录表 update_details（up_det） 更新详细表 update_baidu（up_hot） 更新实时热搜 ''' print(s) else: order = sys.argv[1] if order == "up_his": update_history()