爬虫利器：Python + Xpath Helper插件

# -*- coding: utf-8 -*- import os,time,requests,hashlib requests.DEFAULT_RETRIES = 5 session = requests.session() session.keep_alive = False PATH_IMGS = "images" def download(url): ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" headers = { "user-agent":ua, "accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "proxy-connection":"keep-alive", "Accept-Encoding":"gzip, deflate, br", "Connection":'close', } name = hashlib.md5(url.encode("utf-8")).hexdigest() + ".jpg" proxy = None proxies = { "http":proxy, "https":proxy } r = session.get(url,headers=headers,timeout=20,proxies=proxies, verify=False) with open(fr"{PATH_IMGS}/{name}", "wb")as f: f.write(r.content) def main(): with open(f"C:/Users/18332/Desktop/url列表.txt",encoding="utf-8") as fr: urls0 = [s.strip() for s in fr.readlines() if s.strip()] urls = list(set(urls0)) print(f"待爬取url数目={len(urls)}") for i,url in enumerate(urls): print(f"{i/len(urls)} {url}") # noninspection PyBroadException try: download(url) except: print("skip!") time.sleep(0.5) if __name__ == "__main__": main()