基于python实现博客网页下载并以pdf格式存储

#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2021/8/8 0008 0:10 # @Author : zjl # @File : handle_ryf_article_url.py import re import time import pdfkit import requests from concurrent.futures import ThreadPoolExecutor with open('article_urls', 'r') as fr: article_urls = fr.read() article_urls = article_urls.split('\n') article_headers = { 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'zh-CN,zh;q=0.9,en-GB;q=0.8,en-US;q=0.7,en;q=0.6', 'cache-control': 'max-age=0', 'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'same-origin', 'upgrade-insecure-requests': '1', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36', } def handle_article(article_url): """ 根据获取到的文章url，遍历访问并实现pdf保存 :return: """ resp = requests.get(article_url, headers=article_headers) resp.encoding = 'utf-8' pattern = re.compile(r'<h1 id="page-title" class="asset-name entry-title">(.*)</h1>') article_title = pattern.search(resp.text) if article_title is not None: print(f'标题为{article_title.group(1)}的文章开始下载>>>>>>') options = { 'page-size': 'Letter', 'encoding': "UTF-8", 'custom-header': [('Accept-Encoding', 'gzip')] } try: filename = './article/{}.pdf'.format(article_title.group(1)) path_wk = r'C:\wkhtmltopdf\bin\wkhtmltopdf.exe' # 安装位置 config = pdfkit.configuration(wkhtmltopdf=path_wk) pdfkit.from_string(resp.text, filename, options=options, configuration=config) except: pass finally: print(f'标题为{article_title.group(1)}的文章下载完成<<<<<<') def run(): with ThreadPoolExecutor(max_workers=6) as t: for article_url in article_urls: t.submit(handle_article, article_url) if __name__ == '__main__': start = time.time() run() print(f'complete time: {time.time() - start}')