import requests
from lxml import etree
import time
import random
from bs4 import BeautifulSoup
import re
import jieba
import jieba.analyse
from collections import Counter
import wordcloud as wc
import matplotlib.pyplot as plt
# 获取下一页链接的函数
def next_url(next_url_element):
nxturl = 'https://www.xrqki.cc/books/17474/'
# rfind('/') 获取最后一个'/'字符的索引
index = next_url_element.rfind('/') + 1
nxturl += next_url_element[index:]
return nxturl
# 请求头，需要添加你的浏览器信息才可以运行
headers= {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.0.10191 SLBChan/105"
}
#
# # 小说主页
main_url = "https://www.xrqki.cc/books/17474/"
# # 使用get方法请求网页
main_resp = requests.get(main_url, headers=headers)
# # 将网页内容按utf-8规范解码为文本形式
main_text = main_resp.content.decode('utf-8')
# # 将文本内容创建为可解析元素
main_html = etree.HTML(main_text)
bookTitle = main_html.xpath('/html/body/div[1]/div/div[2]/div/h1/text()')[0]
author = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[1]/span/text()')[0]
update = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[2]/span/text()')[0]
introduction = main_html.xpath('/html/body/div[1]/div/div[2]/div/div[3]/p/text()')[0]
# # 调试期间仅爬取六个页面
# maxPages = 95
cnt = 0
# # 记录上一章节的标题
lastTitle = ''
# # 爬取起点
url = 'https://www.xrqki.cc/books/17474/431556.html'
#
# # 爬取终点
endurl = 'https://www.xrqki.cc/books/17474/431570.html'
while url != endurl:
cnt += 1 # 记录当前爬取的页面
resp = requests.get(url, headers)
html_content=resp.text
soup=BeautifulSoup(html_content,'lxml')
articles=soup.find_all('article')
for atricle in articles:
contents=atricle.get_text()
text = resp.content.decode('utf-8')
html = etree.HTML(text)
title = html.xpath('//*[@id="ss-reader-main"]/div[2]/h1/text()')[0]
# 输出爬取进度信息
print("第 {}页, 标题:{}, 网址: {}".format(cnt, title, url))
with open(bookTitle + '.txt', 'a', encoding='utf-8') as f:
if title != lastTitle: # 章节标题改变
f.write('\t\t\t\t')
f.write(title) # 写入新的章节标题
f.write('\n\n')
lastTitle = title # 更新章节标题
f.write(contents)
f.write('\n\n')
f.close()
# 获取"下一页"按钮指向的链接
next_url_element = html.xpath('//*[@id="next_url"]/@href')[0]
# 传入函数next_url得到下一页链接
url = next_url(next_url_element)
print("爬取完成")
