B站编程课程采集与分析

from PyQt5.QtWidgets import * from PyQt5.QtCore import * from PyQt5.QtGui import * from PyQt5.QtWebEngineWidgets import * import datetime import time from bilibili_UI import * from DB import * import pymysql import requests from lxml import etree from pyecharts import options as opts from pyecharts.charts import Funnel from pyecharts.render import make_snapshot from pyecharts.globals import ThemeType from pyecharts import options as opts from pyecharts.charts import Pie from pyecharts.charts import Bar from pyecharts.charts import WordCloud class DataThread(QThread): signal = pyqtSignal(str, str, str, str, int, int, str) def __init__(self): QThread.__init__(self) self.state = 1 self.page_number = 1 self.o_number = 0 self.key = [] self.value = [] def run(self): # while(self.state): # 爬虫准备工作 base_url = 'https://search.bilibili.com/all?vt=77434542&keyword=%E7%BC%96%E7%A8%8B%E8%AF%BE%E7%A8%8B&from_source=webtop_search&spm_id_from=333.1007&search_source=5' params = {} headers = { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0" } # page_number = 1 # o_number = 0 unique_links = set() video_data_by_keyword = ['C语言', 'C++', 'Python', 'PHP', '算法', 'Java', 'go语言','Mysql','C#','Scratch','web','计算机'] while self.page_number <= 34: params['page'] = str(self.page_number) params['o'] = str(self.o_number) response = requests.get(base_url, params=params, headers=headers) html = response.text html = response.content.decode('utf-8') parse = etree.HTMLParser(encoding='utf-8') contentPath = [] contentname = [] contentauthor = [] contentVV = [] contentCM = [] contentDR = [] doc = etree.HTML(html) doc.xpath('//div[@class="bili-video-card__info--right"]//a/@href') contentPath = doc.xpath('//div[@class="bili-video-card__info--right"]/a/@href') contentname = doc.xpath('//div[@class="bili-video-card__info--right"]//h3[@class="bili-video-card__info--tit"]/@title') contentauthor = doc.xpath('//div[@class="bili-video-card__info--right"]//span[@class="bili-video-card__info--author"]/text()') contentVV = doc.xpath('//div[@class="bili-video-card__stats--left"]/span[@class="bili-video-card__stats--item"][1]/span/text()') contentCM = doc.xpath('//div[@class="bili-video-card__stats--left"]/span[@class="bili-video-card__stats--item"][2]/span/text()') contentDR = doc.xpath('//div[@class="bili-video-card__stats"]/span[@class="bili-video-card__stats__duration"]/text()') # print(contentVV) # print(contentCM) for link, name,auther,vv,cm,dr in zip(contentPath,contentname,contentauthor,contentVV,contentCM,contentDR): category_found = False VideoID = str(self.data) VideoName = name VideoAuther = auther if vv[-1] == '万': num = float(vv[0:-1]) num *= 10000 VideoView = int(num) else: VideoView = int(vv) if cm[-1] == '万': num = float(cm[0:-1]) num *= 10000 Comment = int(num) else: Comment = int(cm) Duration = dr Category = None for keyword in video_data_by_keyword: lower_keyword = keyword.lower() # 将关键词转换为小写 if lower_keyword in name.lower(): Category = keyword if link not in unique_links: if self.state: self.signal.emit(VideoID, VideoName, VideoAuther, Category, VideoView, Comment, Duration) self.data += 1 time.sleep(0.1) unique_links.add(link) break # for keyword in video_data_by_keyword: # if keyword.lower() in name.lower(): # Category = keyword # if self.state: # self.signal.emit(VideoID, VideoName, VideoAuther, Category, VideoView, Comment, Duration) # self.data += 1 # time.sleep(0.2) # break self.page_number += 1 self.o_number += 24 def Stop(self): self.state = 0 class MyWindow(Ui_MainWindow,QMainWindow): signal = pyqtSignal(str, str, str, str, int, int, str) def __init__(self): QMainWindow.__init__(self) self.setupUi(self) self.Operate() self.dt = {'C语言': 0, 'C++': 0, 'Python': 0, 'PHP': 0, '算法': 0, 'Java': 0, 'go语言': 0, 'Mysql': 0, 'C#': 0, 'Scratch': 0, 'web': 0, '计算机': 0} self.showflag = 0 # 初始化并指明各个函数 def Operate(self): self.tabWidget.currentChanged.connect(self.ShowSelWindow) self.tabWidget.setCurrentIndex(0) self.tabWidget1.currentChanged.connect(self.ShowSelWindow) self.tabWidget1.setCurrentIndex(0) self.InitTable() self.ConnectDB() self.CreateThread() self.pButton_data_collection.clicked.connect(self.StartThread) self.pButton_show_four_picture.clicked.connect(self.show_four_htmls) self.pButton_show1.clicked.connect(self.show1) self.pButton_show2.clicked.connect(self.show2) self.pButton_show3.clicked.connect(self.show3) # 初始化各个图表 def InitTable(self): # 设置列个数 self.tableWidget_show_all_datas.setColumnCount(7) # 标题 self.tableWidget_show_all_datas.setHorizontalHeaderLabels( ['视频ID号', '视频名称', '视频作者', '相关分类', '视频观看量', '评论数', '视频时长']) # 设置整行选中模式 self.tableWidget_show_all_datas.setSelectionBehavior(True) # 设置列宽度 self.tableWidget_show_all_datas.setColumnWidth(0, 147) self.tableWidget_show_all_datas.setColumnWidth(1, 630) self.tableWidget_show_all_datas.setColumnWidth(2, 247) self.tableWidget_show_all_datas.setColumnWidth(3, 147) self.tableWidget_show_all_datas.setColumnWidth(4, 120) self.tableWidget_show_all_datas.setColumnWidth(5, 120) self.tableWidget_show_all_datas.setColumnWidth(6, 147) # 最后一列自动填充剩余宽度 # self.tableWidget_show_all_datas.horizontalHeader().setStretchLastSection(True) # 设置标题带排序 self.tableWidget_show_all_datas.setSortingEnabled(True) # 隐藏默认行号 self.tableWidget_show_all_datas.verticalHeader().setHidden(True) def ConnectDB(self): self.con = GetConn() self.cur = self.con.cursor() sql = "delete from videos" try: self.cur.execute(sql) self.con.commit() print('清空数据成功') except Exception as e: print('清空数据失败', e) def ShowSelWindow(self): selindex = self.tabWidget.currentIndex() self.tabWidget.setCurrentIndex(selindex) # 创建多线程 def CreateThread(self): self.datathread = DataThread() self.datathread.data = 1 self.dat