Python快速爬取高德地图poi数据，简单已操作，0基础皆可上手！

#! /usr/local/bin/python3 # coding: utf-8 # __author__ = "Liu jiao" # __date__ = 2019/10/16 16:11 from urllib.parse import quote import json import os import TODO as TODO from transCoordinateSystem import gcj02_to_wgs84, gcj02_to_bd09 import area_boundary as area_boundary import city_grid as city_grid import time import collections import pandas as pd from requests.adapters import HTTPAdapter import requests #from shp import trans_point_to_shp ''' 版本更新说明： 2020.06.19: 1.清除了poi数据写入shp文件相关操作 ''' types2 = ['购物服务', '生活服务', '餐饮服务', '公司企业', '地名地址信息'] for type2 in types2: { ## TODO 1.划分的网格距离，0.02-0.05最佳，建议如果是数量比较多的用0.01或0.02，如餐厅，企业。数据量少的用0.05或者更大，如大学 pology_split_distance = 0.02 ## TODO 2. 城市编码，参见高德城市编码表，注意需要用adcode列的编码 city_code = '440111' ## TODO 3. POI类型编码，类型名或者编码都行，具体参见《高德地图POI分类编码表.xlsx》 typs = ['科教文化服务'] #typs = ['购物服务', '生活服务', '餐饮服务', '公司企业', '地名地址信息'] # ,'科教文化服务','交通设施服务','商务住宅', '政府机构及社会团体', '通行设施', '医疗保健服务', '住宿服务' # , '风景名胜', '公共设施', '金融保险服务', '汽车服务', '汽车维修', '室内设施', '体育休闲服务' # '汽车销售', '摩托车服务','道路附属设施', '事件活动', ## TODO 4. 高德开放平台密钥 gaode_key = [''] # TODO 5.输出数据坐标系,1为高德GCJ20坐标系，2WGS84坐标系，3百度BD09坐标系 coord = 2 ############################################以下不需要动####################################################################### poi_pology_search_url = 'https://restapi.amap.com/v3/place/polygon' buffer_keys = collections.deque(maxlen=len(gaode_key)) def init_queen(): for i in range(len(gaode_key)): buffer_keys.append(gaode_key[i]) print('当前可供使用的高德密钥：', buffer_keys) # 根据城市名称和分类关键字获取poi数据 def getpois(grids, keywords): if buffer_keys.maxlen == 0: print('密钥已经用尽，程序退出！！！！！！！！！！！！！！！') exit(0) amap_key = buffer_keys[0] # 总是获取队列中的第一个密钥 i = 1 poilist = [] while True: # 使用while循环不断分页获取数据 result = getpoi_page(grids, keywords, i, amap_key) print("当前爬取结果:", result) if result != None: result = json.loads(result) # 将字符串转换为json try: if result['count'] == '0': break except Exception as e: print('出现异常：', e) if result['infocode'] == '10001' or result['infocode'] == '10003': print(result) print('无效的密钥！！！！！！！！！！！！！，重新切换密钥进行爬取') buffer_keys.remove(buffer_keys[0]) try: amap_key = buffer_keys[0] # 总是获取队列中的第一个密钥 except Exception as e: print('密钥已经用尽，程序退出...') exit(0) result = getpoi_page(grids, keywords, i, amap_key) result = json.loads(result) hand(poilist, result) i = i + 1 return poilist # 数据写入csv文件中 def write_to_csv(poilist, citycode, classfield, coord): data_csv = {} lons, lats, names, addresss, pnames, citynames,adnames, business_areas, types, typecodes, ids, type_1s, type_2s, type_3s, type_4s = [], [], [], [], [], [], [], [], [], [], [], [], [], [], [] if len(poilist) == 0: print("处理完成，当前citycode:" + str(citycode), ", classfield为：", str(classfield) + "，数据为空，，，结束.......") return None, None for i in range(len(poilist)): location = poilist[i].get('location') name = poilist[i].get('name') address = poilist[i].get('address') pname = poilist[i].get('pname') cityname = poilist[i].get('cityname') adname = poilist[i].get('adname') business_area = poilist[i].get('business_area') type = poilist[i].get('type') typecode = poilist[i].get('typecode') lng = str(location).split(",")[0] lat = str(location).split(",")[1] id = poilist[i].get('id') if (coord == 2): result = gcj02_to_wgs84(float(lng), float(lat)) lng = result[0] lat = result[1] if (coord == 3): result = gcj02_to_bd09(float(lng), float(lat)) lng = result[0] lat = result[1] type_1, type_2, type_3, type_4 = '','','','' if str(type) != None and str(type) != '': type_strs = type.split(';') for i in range(len(type_strs)): ty = type_strs[i] if i == 0: type_1 = ty elif i == 1: type_2 = ty elif i == 2: type_3 = ty elif i == 3: type_4 = ty lons.append(lng) lats.append(lat) names.append(name) addresss.append(address) pnames.append(pname) citynames.append(cityname) adnames.append(adname) if business_area == []: business_area = '' business_areas.append(business_area) types.append(type) typecodes.append(typecode) ids.append(id) type_1s.append(type_1) type_2s.append(type_2) type_3s.append(type_3) type_4s.append(type_4) data_csv['lon'], data_csv['lat'], data_csv['name'], data_csv['address'], data_csv['pname'], \ data_csv['cityname'], data_csv['adname'], data_csv['business_area'], data_csv['type'], data_csv['typecode'], data_csv['id'], data_csv[ 'type1'], data_csv['type2'], data_csv['type3'], data_csv['type4'] = \ lons, lats, names, addresss, pnames, citynames, adnames, business_areas, types, typecodes, ids, type_1s, type_2s, type_3s, type_4s df = pd.DataFrame(data_csv) file_name = 'POI_' + citycode + "-" + classfield + ".csv" df.to_csv(r'data' + os.sep +'POI_' + citycode + "_" + classfield + ".csv" , index=False, encoding='utf_8_sig') #folder_name = 'poi-' + citycode + "-" + classfield #folder_name_full = 'data' + os.sep + folder_name + os.sep #if os.path.exists(folder_name_full) is False: # os.makedirs(folder_name_full) #file_name = 'poi-' + cityco