自动爬取中国大学自动爬取中国大学mooc的的pdf文档文档
由于中国大学mooc里课程的pdf文档不方便下载,因此这里敲一个能自动下载课程里所有pdf的代码:
窗体代码窗体代码:
from tkinter import *
from tkinter.filedialog import askdirectory
from down_main import download
def center_window(tk: Tk, width: int, height: int):
screenwidth = tk.winfo_screenwidth()
screenheight = tk.winfo_screenheight()
size = '%dx%d+%d+%d' % (width, height, (screenwidth - width) / 2, (screenheight - height) / 2)
tk.geometry(size)
tk = Tk()
var = IntVar()
tk.title('my window')
center_window(tk, 800, 300)
Label(tk, text='').pack(anchor=CENTER)
Label(tk, text='课程链接URL').pack(anchor=CENTER)
Label(tk, text='例如: https://www.icourse163.org/learn/....#/learn/content', ).pack(anchor=CENTER)
Label(tk, text='').pack(anchor=CENTER)
url_entry = Entry(tk, width=100)
url_entry.pack(anchor=CENTER)
Label(tk, text='').pack(anchor=CENTER)
def select_save_path():
path = askdirectory()
save_path_label.config(text=path)
Button(tk, text="选择保存的目录", command=select_save_path).pack()
save_path_label = Label(tk, text='选择保存的目录') # 标签的文字
save_path_label.pack()
def begin():
url = url_entry.get()
path = save_path_label.cget("text")
download(url, path)
Label(tk, text='').pack(anchor=CENTER)
Button(tk, text="开始下载", command=begin).pack()
# 主事件循环
mainloop()
爬虫代码爬虫代码:
参考博客:https://blog.csdn.net/weixin_43833642/article/details/105138838
import requests
import re
import threading
import time
import functools
req = requests.session()
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/80.0.3987.132 Safari/537.36',
'content-type': 'text/plain'
}
content_id = [] _id = [] pdf_url = [] threads = [] save_path = ''
def request_mooc_get(url):
return req.get(url=url)
资源评论
