import csv
import sys, time, random, requests
from lxml import etree
import pandas as pd
from run import Ui_Form # 继承run.py
from PyQt5.QtWidgets import *
from PyQt5 import QtCore, QtGui, QtWidgets
from loguru import logger
class MyPyQT_Form(QWidget, Ui_Form):
def __init__(self):
super(MyPyQT_Form, self).__init__()
self.setupUi(self)
def pushButton_click(self):
self.textEdit_2.setText("下载中......")
self.keyword = self.lineEdit.text() # 关键字
self.page = int(self.lineEdit_2.text()) # 页数
# result = pd.DataFrame()
data_list = []
for i in range(1, self.page + 1):
time.sleep(random.random() + 0.5)
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'}
url = 'https://search.bilibili.com/all?keyword={0}&from_source=nav_suggest_new0&page={1}'.format(
self.keyword, self.page)
html = requests.get(url.format(i), headers=headers)
bs = etree.HTML(html.text)
items = bs.xpath('//li[@class = "video-item matrix"]')
for item in items:
title = item.xpath('div[@class = "info"]/div/a/@title')[0]
click = item.xpath('div[@class = "info"]/div[3]/span[1]/text()')[0].strip('\n ').replace("万",
"")
danmu = item.xpath('div[@class = "info"]/div[3]/span[2]/text()')[0].strip('\n ')
date = item.xpath('div[@class = "info"]/div[3]/span[3]/text()')[0].strip('\n ')
up = item.xpath('div[@class = "info"]/div[3]/span[4]/a/text()')[0].strip('\n ')
data_list.append([title, click, danmu, date, up])
data = {'标题': title, '播放量(万)': click, '弹幕': danmu, '日期': date, 'UP主': up}
file_name = '{0}/{1}视频数据.csv'.format(self.download_path, self.keyword)
logger.info('file_name: ', file_name)
logger.info(title, click, danmu, date, up)
with open(file_name, 'a+', encoding='utf_8_sig',
newline='') as fp:
fieldnames = ['标题', '播放量(万)', '弹幕', '日期', 'UP主']
writer = csv.DictWriter(fp, fieldnames=fieldnames)
writer.writerow(data)
self.textEdit_2.append("第{0}页下载完成".format(i))
self.textEdit_2.append("保存路径:{}".format(self.download_path))
self.textEdit_2.append("文件名称:{}".format('{0}共{1}个视频数据.csv'.format(self.keyword, self.page * 20)))
# 创建Pandas DataFrame
df = pd.DataFrame(data_list, columns=['标题', '播放量(万)', '弹幕', '日期', 'UP主'])
# 将DataFrame保存为Excel文件
df.to_excel('{0}/{1}视频数据.xlsx'.format(self.download_path, self.keyword), index=False)
def setBrowerPath(self):
self.download_path = QFileDialog.getExistingDirectory(self)
self.textEdit.setText(self.download_path)
if __name__ == '__main__':
app = QApplication(sys.argv)
my_pyqt_form = MyPyQT_Form()
my_pyqt_form.show()
sys.exit(app.exec_())
