# -*- coding: utf-8 -*-
"""
Created on Mon Jul 20 17:35:39 2020
@author: Administrator
"""
import requests
import os
import time
from lxml import etree
import threading
import random
headers = {
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
}
#存储目录·
root ="webpic/"
#5个大类对应的文件夹
catalog=['','catalog1','catalog2','catalog3','catalog4','catalog5']
#记录已爬取页数
pass_page=['','pass_page1.txt','pass_page2.txt','pass_page3.txt','pass_page4.txt','pass_page5.txt']
"""初始化"""
def initialization():
#初始化爬取记录文本
if not os.path.exists(root):
os.mkdir(root)
for i in range(1,6):
if not os.path.exists(root+pass_page[i]):
open(root+pass_page[i], 'a+')
#初始化图片存储目录
for i in range(1,6):
if not os.path.exists(root+catalog[i]):
os.mkdir(root+catalog[i])
"""当前商品链接"""
def commodity(url,c_whichcatalog):
try:
#请求当前商品链接
r = requests.get(url,headers = headers)
r.encoding='utf-8'
s = r.text
selector = etree.HTML(s)
#解析图片链接
img = selector.xpath('//*[@id="app"]/div/div[1]/main/div/div/div/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div/div[2]/img/@src')
#解析标题
title = selector.xpath('//*[@id="app"]/div/div[1]/main/div/div/div/div[2]/div[1]/div[2]/div/div/div/div[1]/h1/text()')
c = requests.get(img[0],headers = headers)
#保存图片，后缀名为jpg
#判断图片是否存在，不存在则下载
if not os.path.exists(root+catalog[int(c_whichcatalog)]+"/"+title[0]+".jpg"):
with open(root+catalog[int(c_whichcatalog)]+"/"+title[0]+".jpg","wb") as f:
f.write(c.content)
f.close()
except:
#重试再次请求图片链接
try:
#请求当前商品链接
r = requests.get(url,headers = headers)
r.encoding='utf-8'
s = r.text
selector = etree.HTML(s)
#解析图片链接
img = selector.xpath('//*[@id="app"]/div/div[1]/main/div/div/div/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div/div[2]/img/@src')
#解析标题
title = selector.xpath('//*[@id="app"]/div/div[1]/main/div/div/div/div[2]/div[1]/div[2]/div/div/div/div[1]/h1/text()')
c = requests.get(img[0],headers = headers)
#保存图片，后缀名为jpg
#判断图片是否存在，不存在则下载
if not os.path.exists(root+catalog[int(c_whichcatalog)]+"/"+title[0]+".jpg"):
with open(root+catalog[int(c_whichcatalog)]+"/"+title[0]+".jpg","wb") as f:
f.write(c.content)
f.close()
except:
pass
"""当前大类链接"""
def categories(url,whichcatalog,page):
try:
#请求当前大类链接
r = requests.get(url,headers = headers)
r.encoding='utf-8'
s = r.text
selector = etree.HTML(s)
#遍历当前页的所有商品
for item in selector.xpath('//*[@id="SearchResultsGrid"]/a'):
href = item.xpath('.//@href')
print(href[0])
#多线程下载图片
threading.Thread(target=commodity,args=(href[0],whichcatalog,)).start()
#随机延时1~2秒，防止网站禁止ip
#time.sleep(int(random.randint(1,2)))
#如果网速稳定可以注释掉time.sleep(2)
#time.sleep(2)
#commodity(href[0],whichcatalog)
#记录当前爬取页数
with open(root+pass_page[int(whichcatalog)],"a+",encoding='utf8') as f:
f.writelines(str(page))
f.writelines("\n")
except:
pass
def start(whichcata):
#当前爬取的页数
current_page=1
while(True):
with open(root+pass_page[int(whichcata)],"r") as f: #设置文件对象
pagelist = f.read() #可以是随便对文件的操作
#判断当前页数是否被爬取
if str(current_page) not in pagelist:
if whichcata==1:
categories("https://www.redbubble.com/shop/*?iaCode=u-tees&page="+str(current_page)+"&searchType=find&tShirtColor=tShirtColor-black",1,current_page)
elif whichcata==2:
categories("https://www.redbubble.com/shop/*?iaCode=u-tees&page="+str(current_page)+"&searchType=find&tShirtColor=tShirtColor-white",2,current_page)
elif whichcata==3:
categories("https://www.redbubble.com/shop/stickers?page="+str(current_page)+"&searchType=browse",3,current_page)
elif whichcata==4:
categories("https://www.redbubble.com/shop/cloth+face+masks?page="+str(current_page)+"&searchType=browse",4,current_page)
elif whichcata==5:
categories("https://www.redbubble.com/shop/phone-cases?page="+str(current_page)+"&searchType=browse",5,current_page)
current_page=current_page+1
if __name__ == '__main__':
"""初始化"""
initialization()
#指定大类
whichcata=input("请输入数字(1~5),指定爬取哪一个大类:")
#开始爬取
print("开始爬取第"+str(whichcata)+"大类.....")
start(int(whichcata))
