空气质量爬虫以及可视化_空气质量爬虫_全国空气质量爬虫以及可视化_

#全国空气质量的爬虫设计 import requests from bs4 import BeautifulSoup import bs4 import re aqilist = [] # 储存城市AQI clist = [] # 储存城市链接 cnlist = [] # 储存城市名字 cwlink = [ "/air/kezilesuzhou/", "/air/simao/", "/air/xiangfan/","/air/naqudiqu/"] # 异常链接 def get_one_page(city): # 获得HTML 爬取城市信息 url = "http://www.air-level.com" + city #各个城市链接 if city in cwlink: #排除异常 aqilist.append("异常链接") else: try: kv = {'user-agent': 'Mozilla/5.0'} # 伪装成浏览器，headers r = requests.get(url, headers=kv) #使用get方法打开各个城市的链接 r.raise_for_status() #如果返回的状态码不是200，则抛出异常 r.encoding = r.apparent_encoding #判断网页的编码格式 ，便于r.text知道如何解码 except: print("爬取失败") demo = r.text #响应url对应的页面内容 soup = BeautifulSoup(demo, "html.parser")#解析页面内容 s = soup.find("span") #获取span中的内容 aqilist.append(s.string) #将span中的内容以字符串的形式添加到列表aqilist def get_all_city(): # 爬取城市链接 url = "http://www.air-level.com" #空气知音链接 try: kv = {'user-agent': 'Mozilla/5.0'} # 伪装成浏览器，headers r = requests.get(url, headers=kv) #通过get方法打开空气知音链接，返回Response对象 r.raise_for_status() #如果返回的状态码不是200，则抛出异常 r.encoding = r.apparent_encoding #更改编码为r.apparent_encoding，windows-1254或utf-8 except: print("爬取城市链接失败") demo = r.text #响应url对应的页面内容 soup = BeautifulSoup(demo, "html.parser") time = soup.find('h4').string #获取数据更新时间 print(time) for it in soup.find(id="citylist").children:#children可以列出所有的子类 if isinstance(it, bs4.element.Tag): # 检测it的类型，得是一个bs4.element.Tag类型 for its in it.find_all('a'): clist.append(its.get('href')) # 加入列表当中去 cnlist.append(its.string) #将href中的内容以字符串的形式添加到列表cnlist def main(): fo = open("air.csv","w") get_all_city() print("共爬取了{}个城市".format(len(clist))) print("{1:{0}^7}{2:{0}^7}{3:{0}^7}".format(chr(12288),"城市","AQI","空气质量等级")) air = ["城市","AQI","空气质量等级"] fo.write(",".join(air)+"\n") for it in range(len(clist)): get_one_page(clist[it]) if aqilist[it] == "异常链接" or aqilist[it]== "-":#删除列表中的异常输出 aqilist[it] = "" cnlist[it] = "" else: airaqi = 0 airlevel = "" pattern1 = re.compile(r'\d+') #找到字符串中的数字 pattern2 = re.compile(r'\D+') airaqi0 = [re.findall(pattern1,aqilist[it])] #将字符串中的数字返回为列表，赋值给airaqi0 for i in airaqi0: airaqi = i[0] #获取列表第一个元素，列表只有一个元素 airlevel0 = [re.findall(pattern2,aqilist[it])] for i in airlevel0: airlevel = i[0] air = [cnlist[it],airaqi,airlevel] #定义一个存储数据的二维数组 fo.write(",".join(air)+"\n") print("{1:{0}^7}{2:{0}^8}{3:{0}^7}".format(chr(12288),cnlist[it], airaqi,airlevel)) fo.close() main()