python爬虫 豆瓣电影Top250数据分析与可视化（应用Flask框架、Echarts、WordCloud等技术）

# -*- coding: UTF-8 -*- # @Time : 2023/7/13 18:45 # @Author : lwy # @File : spider.py from bs4 import BeautifulSoup # 网页解析获取数据 import re # 正则表达式，进行文字匹配 import urllib.request, urllib.error # 制定url import xlwt # 进行excel操作 import sqlite3 # 进行数据库操作 def main(): baseurl = "https://movie.douban.com/top250?start=" # 1.爬取网页 datalist = getData(baseurl) # 2.逐一解析数据 # 3.保存数据 # savepath = ".\\豆瓣电影Top250.xls" # saveData(datalist, savepath) dbpath = "movie.db" saveData2DB(datalist, dbpath) # askURL("https://movie.douban.com/top250?start=0") findLink = re.compile(r'<a href="(.*?)">') # 创建正则表达式对象 findImgSrc = re.compile(r'<img.*src="(.*?)"', re.S) # re.S让换行符包含在匹配中 findTitle = re.compile(r'<span class="title">(.*)</span>') findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>') findJudge = re.compile(r'<span>(\d*)人评价</span>') # 概况 findInq = re.compile(r'<span class="inq">(.*)</span>') # 影片相关内容 findBd = re.compile(r'<p class="">(.*?)</p>', re.S) def getData(baseurl): datalist = [] for i in range(0, 10): url = baseurl + str(i * 25) html = askURL(url) # 2.逐一解析数据 soup = BeautifulSoup(html, "html.parser") for item in soup.find_all('div', class_="item"): data = [] item = str(item) link = re.findall(findLink, item)[0] # 影片详情链接 data.append(link) imgSrc = re.findall(findImgSrc, item)[0] data.append(imgSrc) titles = re.findall(findTitle, item) # 可能只有一个名字 if (len(titles) == 2): ctitle = titles[0] data.append(ctitle) otitle = titles[1].replace("/", "") data.append(otitle) else: data.append(titles[0]) data.append(' ') # 外国名留空 rating = re.findall(findRating, item)[0] data.append(rating) judgeNum = re.findall(findJudge, item)[0] data.append(judgeNum) inq = re.findall(findInq, item) if len(inq) != 0: inq = inq[0].replace("。", "") data.append(inq) else: data.append(" ") bd = re.findall(findBd, item)[0] bd = re.sub('<br(\s+)?/>(\s+)?', "", bd) # 去掉<br/> bd = re.sub('/', "", bd) # 替换/ data.append(bd.strip()) # 去掉前后空格 datalist.append(data) # 处理好的一部电影 return datalist # 得到指定一个URL的网页内容 def askURL(url): head = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" } request = urllib.request.Request(url, headers=head) html = "" try: response = urllib.request.urlopen(request) html = response.read().decode("utf-8") # print(html) except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) if hasattr(e, "reason"): print(e.reason) return html def saveData(datalist, savepath): print("save...") book = xlwt.Workbook(encoding="utf-8", style_compression=0) sheet = book.add_sheet("豆瓣电影Top250", cell_overwrite_ok=True) col = ("电影详情链接", "图片链接", "影片中文名", "影片外文名", "评分", "评价数", "概况", "相关信息") for i in range(0, 8): sheet.write(0, i, col[i]) for i in range(0, 250): print("第%d条" % i) data = datalist[i] for j in range(0, 8): sheet.write(i + 1, j, data[j]) book.save(savepath) def saveData2DB(datalist, dbpath): init_db(dbpath) conn = sqlite3.connect(dbpath) cur = conn.cursor() for data in datalist: for index in range(len(data)): if index == 4 or index == 5: continue data[index] = '"' + data[index] + '"' sql = ''' insert into movie250( info_link, pic_link, cname, ename, score, rated, introduction, info) values(%s) ''' % ",".join(data) cur.execute(sql) conn.commit() cur.close() conn.close() def init_db(dbpath): sql = ''' create table movie250 ( id integer primary key autoincrement, info_link text, pic_link text, cname varchar, ename varchar, score numeric, rated numeric, introduction text, info text ) ''' conn = sqlite3.connect(dbpath) cursor = conn.cursor() cursor.execute(sql) conn.commit() conn.close() print("init success") if __name__ == "__main__": # 调用函数 main() print("爬取完毕")