python爬取微博关键词搜索博文,修改cookie和地址就可以

"""__author__:Orange """ import time from collections import Counter import jieba from lxml import etree import requests # 爬虫获取 def get_Research(research_Words, page): headers = { 'Host': 's.weibo.com', 'Cookie': 'SINAGLOBAL=9144757084286.572.1603873591535; _s_tentry=login.sina.com.cn; ' 'Apache=2106463437790.802.1615947115553; ' 'ULV=1615947115561:77:3:3:2106463437790.802.1615947115553:1615863469171; ' 'login_sid_t=1ad4e167c46fa331cac8e9d1bad5b73e; cross_origin_proto=SSL; UOR=,,login.sina.com.cn; ' 'appkey=; ALF=1647483366; SSOLoginState=1615947366; ' 'SCF=ApYThqk4916OrNn9zNxwX9sLzFvlD13IgPQNWd-9V1VJ2LDeOG0Z8et2m6UvPvLRuNDb-9-ChEpWDzuwkGgDDv0.; ' 'SUB=_2A25NVRY5DeRhGeFL7FUZ8y_Iyj2IHXVuIwDxrDV8PUNbmtANLU79kW9NfboBvxBSoJBysiqz60EsbjMTrp8pshSP; ' 'SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W51slVN9dElVUdLTaSqNo_j5JpX5KzhUgL' '.FoMfS0MRe02XeK22dJLoIp7LxKML1KBLBKnLxKqL1hnLBoMf1KBNeoBfS024; wvr=6; ' 'webim_unReadCount=%7B%22time%22%3A1615947417491%2C%22dm_pub_total%22%3A0%2C%22chat_group_client%22' '%3A0%2C%22chat_group_notice%22%3A0%2C%22allcountNum%22%3A1%2C%22msgbox%22%3A0%7D; ' 'WBStorage=8daec78e6a891122|undefined', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/86.0.4240.111 Safari/537.36 ' } url = f'https://s.weibo.com/weibo?q={research_Words}&wvr=6&b=1&Refer=SWeibo_box&page={page}' # print(url) # print(urlencode(params)) # 异常捕获错误 try: response = requests.get(url, headers=headers) time.sleep(0.1) response.encoding = 'utf-8' if response.status_code == 200: weibo_data = '' html_result = response.text data = etree.HTML(html_result) data = data.xpath('//*[@node-type="feed_list_content"]/text()') # xpath定位元素获取text内容 for i in data: # 处理list中存在的回车空格 i.replace(' ', '') i.replace('\n', '').replace('\r', '') weibo_data += i return weibo_data except requests.ConnectionError: print(None) # 创建停用词list def stopwordslist(filepath): stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()] return stopwords # 对句子分词 def seg_sentence(sentence): sentence_seged = jieba.cut(sentence.strip()) path = r'D:\新建文件夹\稻米数据\Python-根据关键词抓取微博数据再生成词云\weibo_wordcloud-master\stopwords.txt' stopwords = stopwordslist(path) # 这里加载停用词的路径 outstr = '' for word in sentence_seged: if word not in stopwords: if word != '\t': outstr += word outstr += " " return outstr def main(word, weibo_page): weibo_data = '' for i in range(1, 51): result = get_Research(word, weibo_page) # 获取所有词 weibo_data += result return weibo_data # 主函数入口 if __name__ == '__main__': name = input('请输入需要爬取的关键词:') page = input('请输入爬取页数:') data = main(name, page) # 保存原始数据文件 with open(f'weibo_{name}.txt', 'w', encoding='utf-8') as f: f.write(data) data = seg_sentence(data) deal_data = dict(Counter(data)) # 保存词频处理后文件 with open(f'weibo_deal_{name}.txt', 'w', encoding='utf-8') as foo: # 保存到文件中 for k, v in deal_data.items(): foo.write('%s,%d\n' % (k, v))