import re
import time
import requests
import pandas as pd
from bs4 import BeautifulSoup
# 反爬虫
cookie = {
'cookie_name':'' # 这里填你的登录cookie，不填的话只能爬200条
}
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE'
}
span_txt = []
rating = []
location = []
comment_time = []
# 获取3页的影评
pages = 30
for page in range(1, pages+1):
# 输入对应电影的短评网址
res = requests.get(
# 请参照这个格式填写你要爬取的网址
f"https://movie.douban.com/subject/26631790/comments?start={page*20}&limit=20&status=P&sort=new_score",
headers=headers, cookies=cookie)
bs_4 = BeautifulSoup(res.text, 'html.parser')
# 找到所有的最上级
top_class = bs_4.find_all("div", class_="comment-item")
# 获取影评数据
for each_class in top_class:
# 获取短评文本
span_txt.append(each_class.select('span[class=short]')[0].text)
# 获取评价等级
rating_element = each_class.select_one('.rating[title]')
if rating_element:
rating_text = rating_element['title']
rating.append(rating_text)
else:
rating.append('N/A') # 添加默认值或特定标记来表示未找到评分信息
# 获取IP地址
location_span = each_class.select_one('span.comment-location')
location_text = location_span.text
location.append(location_text)
# 获取时间
commentTime_span = each_class.select_one('span.comment-time')
commentTime_text = re.sub(r"\s+|\d{2}:\d{2}:\d{2}", "", commentTime_span.text)
comment_time.append(commentTime_text.replace("-", " - ").replace(" ", ""))
# Delay
time.sleep(0.2)
# 字典中的key值即为csv中列名
# 将数据保存到csv里
dataframe = pd.DataFrame({'短评': span_txt, '评价等级': rating, '用户来自地区': location, '评论时间': comment_time})
# 将DataFrame存储为csv,index表示是否显示行名，default=True
dataframe.to_csv("movie_comments.csv", index=False, sep=',', encoding='utf-8', date_format='%Y-%m-%d')
with open("movie_comments.csv", "w", encoding='utf-8-sig', newline='') as file:
dataframe.to_csv(file, index=False, sep=',')
