利用python语言爬取网站小说资源并利用word cloud 和jieba分词分析得出词云

import requests from lxml import etree import time import random from bs4 import BeautifulSoup import re import jieba import jieba.analyse from collections import Counter import wordcloud as wc import matplotlib.pyplot as plt # 获取下一页链接的函数 def next_url(next_url_element): nxturl = 'https://www.xrqki.cc/books/17474/' # rfind('/') 获取最后一个'/'字符的索引 index = next_url_element.rfind('/') + 1 nxturl += next_url_element[index:] return nxturl # 请求头，需要添加你的浏览器信息才可以运行 headers= { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.0.10191 SLBChan/105" } # # # 小说主页 main_url = "https://www.xrqki.cc/books/17474/" # # 使用get方法请求网页 main_resp = requests.get(main_url, headers=headers) # # 将网页内容按utf-8规范解码为文本形式 main_text = main_resp.content.decode('utf-8') # # 将文本内容创建为可解析元素 main_html = etree.HTML(main_text) bookTitle = main_html.xpath('/html/body/div[1]/div/div[2]/div/h1/text()')[0] author = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[1]/span/text()')[0] update = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[2]/span/text()')[0] introduction = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[3]/p/text()')[0] # # 调试期间仅爬取六个页面 # maxPages = 95 cnt = 0 # # 记录上一章节的标题 lastTitle = '' # # 爬取起点 url = 'https://www.xrqki.cc/books/17474/431556.html' # # # 爬取终点 endurl = 'https://www.xrqki.cc/books/17474/431570.html' while url != endurl: cnt += 1 # 记录当前爬取的页面 resp = requests.get(url, headers) html_content=resp.text soup=BeautifulSoup(html_content,'lxml') articles=soup.find_all('article') for atricle in articles: contents=atricle.get_text() text = resp.content.decode('utf-8') html = etree.HTML(text) title = html.xpath('//*[@id="ss-reader-main"]/div[2]/h1/text()')[0] # 输出爬取进度信息 print("第 {}页, 标题:{}, 网址: {}".format(cnt, title, url)) with open(bookTitle + '.txt', 'a', encoding='utf-8') as f: if title != lastTitle: # 章节标题改变 f.write('\t\t\t\t') f.write(title) # 写入新的章节标题 f.write('\n\n') lastTitle = title # 更新章节标题 f.write(contents) f.write('\n\n') f.close() # 获取"下一页"按钮指向的链接 next_url_element = html.xpath('//*[@id="next_url"]/@href')[0] # 传入函数next_url得到下一页链接 url = next_url(next_url_element) print("爬取完成")