import json
import os.path
import re
import time
import requests
from lxml import etree
# 需要改headers的cookie、referer以及params中的token
from ParaLib import headers, params, url
import warnings
from USER_DATA import USER_DATA_LIST
warnings.filterwarnings('ignore')
global USERINDEX
USERINDEX = 1
timestemp = int(time.time() * 1000)
headers['referer'] = headers['referer'].format(USER_DATA_LIST[USERINDEX]['token'], timestemp + 32)
headers['cookie'] = re.compile('[|][0-9]{13}[^0-9][|]').sub(f'|{timestemp}|', USER_DATA_LIST[USERINDEX]['cookie'])
params['token'] = USER_DATA_LIST[USERINDEX]['token']
headers['user-agent'] = USER_DATA_LIST[USERINDEX]['user-agent']
def makejson(dic):
lis = []
dic['publish_page'] = json.loads(dic['publish_page'])
for i, li in enumerate(dic['publish_page']['publish_list']):
dic['publish_page']['publish_list'][i]['publish_info'] = json.loads(
dic['publish_page']['publish_list'][i]['publish_info'])
for nj in dic['publish_page']['publish_list']:
for nm in nj['publish_info']['appmsgex']:
lis.append({'title': nm['title'], 'link': nm['link'], 'createtime': nm['create_time']})
return lis
with open('FakeIdList.json', 'r', encoding='utf-8') as f:
fakeid = json.load(f)
f.close()
def changeUser():
global USERINDEX
USERINDEX = (USERINDEX + 1) % len(USER_DATA_LIST)
timestemp = int(time.time() * 1000)
headers['cookie'] = re.compile('[|][0-9]{13}[^0-9][|]').sub(f'|{timestemp}|', USER_DATA_LIST[USERINDEX]['cookie'])
headers['referer'] = headers['referer'].format(USER_DATA_LIST[USERINDEX]['token'], timestemp + 32)
params['token'] = USER_DATA_LIST[USERINDEX]['token']
headers['user-agent'] = USER_DATA_LIST[USERINDEX]['user-agent']
# PAGE=[327 ,443 ,740 ,430 ,205 ,487 ,388]
PAGE = [300, 400, 700, 400, 200, 450, 360]
iii = 1
for fak in fakeid[1:]:
print("开始爬取公众号：", fak['name'])
# 对于不同的公众号需要更改params中的fakeid
params['fakeid'] = fak['fakeid']
if iii == 1:
start = 10
else:
start = 0
for num in range(start, PAGE[iii]):
# params中的begin=0:第一页的5天的文章，5:第二页的5天的文章
params['begin'] = num * 5
print("get Page", num + 1)
web_data = requests.get(url, headers=headers, params=params, verify=False)
errorsum = 0
while True:
try:
pagelist = makejson(web_data.json())
break
except:
print("user:", params['token'], "封禁")
errorsum += 1
if errorsum % len(USER_DATA_LIST) == 0:
print("用户池耗尽，休息0.5个小时后再爬")
time.sleep(60 * 60 * 0.5)
changeUser()
print("切换用户信息", params['token'])
web_data = requests.get(url, headers=headers, params=params, verify=False)
# pagelist = makejson(web_data.json())
# 获取该页5天全部文章的标题和链接并返回列表对象
print(pagelist)
for i, page in enumerate(pagelist):
webdata = requests.get(page['link'])
webdata.encoding = 'utf-8'
tree = etree.HTML(webdata.text)
title = tree.xpath('//*[@id="activity-name"]/text()')
createtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(page['createtime']))
content = tree.xpath('//*[@id="js_content"]//p//text()')
filebasename = "data/" + fak['name'] + '/' + createtime[:10] + '/'
if not os.path.exists(filebasename):
os.makedirs(filebasename)
with open(filebasename + f"{len(os.listdir(filebasename)) + 1}" + '.txt', 'w', encoding='utf-8') as f:
f.writelines(''.join(title).strip())
f.write('\n')
f.writelines(createtime)
f.write('\n')
f.writelines(''.join(content).replace('\n', '').strip())
f.close()
# ---------------------------save image-----------------------------
content = tree.xpath('//*[@id="js_content"]//p/img/@data-src')
picbasename = filebasename.replace('data', 'pic')
if not os.path.exists(picbasename):
os.makedirs(picbasename)
for j, piclink in enumerate(content):
with open(picbasename + f"{len(os.listdir(filebasename))}" + "-" + str(j + 1) + ".jpg", 'wb') as f:
f.write(requests.get(piclink).content)
f.close()
# ---------------------------save image over-----------------------------
time.sleep(0.5)
time.sleep(1)
time.sleep(1)
iii += 1
