微博评论.zip

# -*- coding: utf-8 -*- # @Time : 2021/12/8 10:20 # @Author : MinChess # @File : weibo.py # @Software: PyCharm import requests from lxml import etree from urllib.parse import quote import csv import re import time import random from html.parser import HTMLParser headers_com = { 'Cookie': 'SINAGLOBAL=8530308723572.626.1594801309423; UOR=,,www.baidu.com; ALF=1649247528; SUB=_2A25PIYh4DeRhGeVJ7lQS8C_Oyz-IHXVs7SgwrDV8PUJbkNAKLU7kkW1NT8v1zHG7ndkzD92IXpMb9pyYayxZv5p-; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9Wh0wcVpufLRVzXYfa3pjY.Q5JpX5oz75NHD95Q0S0-ce05peo50Ws4Dqcjci--fi-i2i-zpi--fi-82iK.7i--Xi-zRiKy2i--Ri-88i-zRi--fiKyWi-2Ri--Xi-zRi-zc; ULV=1646658060503:5:1:1:243708356859.10474.1646658060493:1627299580145; WBPSESS=--AcJbA8Oa9zAFKPUTSD5sLC7tlhjwiMerLK05g7m8lHcPohg3YiE1uguj3KKHQSomGyiIUaCIxDcQ8SorKYE4ke6D9-V88QEw1v2kXwKPZzIIKl9DmiSwzsb_sbtUU6Qjd2cgrlnOdjlrSBRc2ayQ==; XSRF-TOKEN=6W6_mdFm7HVugNLWYT4De_m1', # 'Cookie': 'SINAGLOBAL=6692839056672.04.1575211969653; XSRF-TOKEN=dvRQl6BFSaBrVQ9aoz2c-lau; _s_tentry=weibo.com; Apache=3761453327801.136.1646320550576; ULV=1646320550596:1:1:1:3761453327801.136.1646320550576:; UOR=,,www.baidu.com; login_sid_t=c055241708da7f3454da76e72babbf12; cross_origin_proto=SSL; ALF=1678169320; SCF=ApwEkyMNH24hMtFSKVLzEW_0pBDzpWweLRBt1V_aCUAGg_823TfLvw0CHZ5_j8J4itdbpBNo1i3scT2gChOhJEI.; WBPSESS=bwbIVYtIkNrJPKlhfJ1TLTI4ZrCYeqURLSWwfQ5QY1_xCzsPzZlG2luc9qbEWVTtt5VJEkReleyuvjb8g-Jcu6vCjNqx2-d46YFyR7feEIwjYWLqWU7e4c64BX02U-ujI5rXPTsoipPuP6cTVWkBFQ==; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWy-ExKsYH4ouv2pp40B4Md5NHD95QESoBcSoBXe0.pWs4Dqcj8TK84Ug2t; SSOLoginState=1646659617; SUB=_2A25PInhxDeRhGeRI71QX9CrPwj2IHXVs7Rg5rDV8PUJbkNB-LW3fkW1NUrg5CZibBb5RX_J1BdEiJc81kTpMpuEb', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36' } headers_cn = { 'Cookie': 'SUB=_2A25PIYh3DeRhGeVJ7lQS8C_Oyz-IHXVs7Sg_rDV6PUJbkdAKLU2nkW1NT8v1zA4Q0lakelzRb0x1ELOzOBSmUcOp; SCF=AtH4DCXADZs6FfnqoYSmwpUovXDFEGtPUyLpA8VSmAi8gymQXfWethijtJEZtyxd0rxS2BSamrES0Z3AQaim2FQ.; _T_WM=16923311902; WEIBOCN_FROM=1110006030; MLOGIN=1; M_WEIBOCN_PARAMS=uicode%3D20000174', # 'Cookie': '_T_WM=48060870584; BAIDU_SSP_lcr=https://security.weibo.com/; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWy-ExKsYH4ouv2pp40B4Md5NHD95QESoBcSoBXe0.pWs4Dqcj8TK84Ug2t; SCF=ApwEkyMNH24hMtFSKVLzEW_0pBDzpWweLRBt1V_aCUAGx_5t8MsvZWG0xU0wbpvlyHnR-BdbVzll0Tm4JqNmyzs.; SUB=_2A25PInhxDeRhGeRI71QX9CrPwj2IHXVs7Rg5rDV6PUJbktCOLRTYkW1NUrg5CV_Fr49NHRIZpmpCqqeqJSJ0Dpol; SSOLoginState=1646659617', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36' } baseUrl = 'https://s.weibo.com/weibo?q={}&Refer=index' topic = '郑州暴雨' csvfile = open(topic + '.csv', 'a', newline='', encoding='utf-8-sig') writer = csv.writer(csvfile) def getTopic(url): page = 0 pageCount = 1 while True: weibo_content = [] weibo_liketimes = [] weibo_date = [] page = page + 1 tempUrl = url + '&page=' + str(page) print('-' * 36, tempUrl, '-' * 36) response = requests.get(tempUrl, headers=headers_com) html = etree.HTML(response.text, parser=etree.HTMLParser(encoding='utf-8')) count = len(html.xpath('//div[@class="card-wrap"]')) - 2 for i in range(1, count + 1): try: contents = html.xpath('//div[@class="card-wrap"][' + str( i) + ']/div[@class="card"]/div[1]/div[2]/p[@node-type="feed_list_content_full"]') contents = contents[0].xpath('string(.)').strip() # 读取该节点下的所有字符串 except: contents = html.xpath('//div[@class="card-wrap"][' + str( i) + ']/div[@class="card"]/div[1]/div[2]/p[@node-type="feed_list_content"]') # 如果出错就代表当前这条微博有问题 try: contents = contents[0].xpath('string(.)').strip() except: continue contents = contents.replace('收起全文d', '') contents = contents.replace('收起d', '') contents = contents.split(' 2')[0] # 发微博的人的名字 name = \ html.xpath( '//div[@class="card-wrap"][' + str(i) + ']/div[@class="card"]/div[1]/div[2]/div[1]/div[2]/a')[ 0].text # 微博url weibo_url = html.xpath( '//div[@class="card-wrap"][' + str(i) + ']/div[@class="card"]/div[1]/div[2]/p[@class="from"]/a/@href')[ 0] url_str = '.*?com\/\d+\/(.*)\?refer_flag=\d+_' res = re.findall(url_str, weibo_url) weibo_url = res[0] host_url = 'https://weibo.cn/comment/' + weibo_url # 发微博的时间 timeA = \ html.xpath( '//div[@class="card-wrap"][' + str(i) + ']/div[@class="card"]/div[1]/div[2]/p[@class="from"]/a')[ 0].text.strip() # 点赞数 likeA = html.xpath( '//div[@class="card-wrap"][' + str(i) + ']/div[@class="card"]/div[2]/ul[1]/li[3]/a/button/span[2]')[ 0].text hostComment = \ html.xpath('//div[@class="card-wrap"][' + str(i) + ']/div[@class="card"]/div[2]/ul[1]/li[2]/a')[0].text # 如果点赞数为空，那么代表点赞数为0 if likeA == '赞': likeA = 0 if hostComment == '评论 ': hostComment = 0 if hostComment != 0: print('正在爬取第', page, '页，第', i, '条微博的评论。') getComment(host_url) try: hosturl, host_sex, host_location, hostcount, hostfollow, hostfans = getpeople(name) list = ['微博', name, hosturl, host_sex, host_location, hostcount, hostfollow, hostfans, contents, timeA, likeA] writer.writerow(list) except: continue print('=' * 66) try: if pageCount == 1: pageA = html.xpath('//*[@id="pl_feedlist_index"]/div[5]/div/a')[0].text print(pageA) pageCount = pageCount + 1 elif pageCount == 50: print('没有下一页了') break else: pageA = html.xpath('//*[@id="pl_feedlist_index"]/div[5]/div/a[2]')[0].text pageCount = pageCount + 1 print(pageA) except: print('没有下一页了') break def getpeople(name): findPoeple = 0 url2 = 'https://s.weibo.com/user?q=' while True: try: response = requests.post('https://weibo.cn/search/?pos=search', headers=headers_cn, data={'suser': '找人', 'keyword': name}) tempUrl2 = url2 + quote(str(name)) + '&Refer=weibo_user' print('搜索页面', tempUrl2) response2 = requests.get(tempUrl2, headers=headers_com) html = etree.HTML(response2.content, parser=etree.HTMLParser(encoding='utf-8')) hosturl_01 = html.xpath('/html/body/div[1]/div[2]/div/div[2]/div[1]/div[3]/div[1]/div[2]/div/a/@href')[0] url_str = '.*?com\/(.*)' res = re.findall(url_str, hosturl_01) hosturl = 'https://weibo.cn/' + res[0] print('找人主页：', hosturl) break except: if findPoeple == 10: stop = random.randint(60, 300) print('IP被封等待一段时间在爬', stop, '秒') time.sleep(stop) if response.status_code == 200: return print('找人') time.sleep(random.randint(0, 10)) findPoeple =