import json
import os.path
import re
import time
import requests
from lxml import etree
# 需要改headers的cookie、referer以及params中的token
from ParaLib import headers, params, url
import warnings
from USER_DATA import USER_DATA_LIST
warnings.filterwarnings('ignore')
global USERINDEX
USERINDEX = 1
timestemp = int(time.time() * 1000)
headers['referer'] = headers['referer'].format(USER_DATA_LIST[USERINDEX]['token'], timestemp + 32)
headers['cookie'] = re.compile('[|][0-9]{13}[^0-9][|]').sub(f'|{timestemp}|', USER_DATA_LIST[USERINDEX]['cookie'])
params['token'] = USER_DATA_LIST[USERINDEX]['token']
headers['user-agent'] = USER_DATA_LIST[USERINDEX]['user-agent']
def makejson(dic):
lis = []
dic['publish_page'] = json.loads(dic['publish_page'])
for i, li in enumerate(dic['publish_page']['publish_list']):
dic['publish_page']['publish_list'][i]['publish_info'] = json.loads(
dic['publish_page']['publish_list'][i]['publish_info'])
for nj in dic['publish_page']['publish_list']:
for nm in nj['publish_info']['appmsgex']:
lis.append({'title': nm['title'], 'link': nm['link'], 'createtime': nm['create_time']})
return lis
with open('FakeIdList.json', 'r', encoding='utf-8') as f:
fakeid = json.load(f)
f.close()
def changeUser():
global USERINDEX
USERINDEX = (USERINDEX + 1) % len(USER_DATA_LIST)
timestemp = int(time.time() * 1000)
headers['cookie'] = re.compile('[|][0-9]{13}[^0-9][|]').sub(f'|{timestemp}|', USER_DATA_LIST[USERINDEX]['cookie'])
headers['referer'] = headers['referer'].format(USER_DATA_LIST[USERINDEX]['token'], timestemp + 32)
params['token'] = USER_DATA_LIST[USERINDEX]['token']
headers['user-agent'] = USER_DATA_LIST[USERINDEX]['user-agent']
# PAGE=[327 ,443 ,740 ,430 ,205 ,487 ,388]
PAGE = [300, 400, 700, 400, 200, 450, 360]
iii = 1
for fak in fakeid[1:]:
print("开始爬取公众号:", fak['name'])
# 对于不同的公众号需要更改params中的fakeid
params['fakeid'] = fak['fakeid']
if iii == 1:
start = 10
else:
start = 0
for num in range(start, PAGE[iii]):
# params中的begin=0:第一页的5天的文章,5:第二页的5天的文章
params['begin'] = num * 5
print("get Page", num + 1)
web_data = requests.get(url, headers=headers, params=params, verify=False)
errorsum = 0
while True:
try:
pagelist = makejson(web_data.json())
break
except:
print("user:", params['token'], "封禁")
errorsum += 1
if errorsum % len(USER_DATA_LIST) == 0:
print("用户池耗尽,休息0.5个小时后再爬")
time.sleep(60 * 60 * 0.5)
changeUser()
print("切换用户信息", params['token'])
web_data = requests.get(url, headers=headers, params=params, verify=False)
# pagelist = makejson(web_data.json())
# 获取该页5天全部文章的标题和链接并返回列表对象
print(pagelist)
for i, page in enumerate(pagelist):
webdata = requests.get(page['link'])
webdata.encoding = 'utf-8'
tree = etree.HTML(webdata.text)
title = tree.xpath('//*[@id="activity-name"]/text()')
createtime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(page['createtime']))
content = tree.xpath('//*[@id="js_content"]//p//text()')
filebasename = "data/" + fak['name'] + '/' + createtime[:10] + '/'
if not os.path.exists(filebasename):
os.makedirs(filebasename)
with open(filebasename + f"{len(os.listdir(filebasename)) + 1}" + '.txt', 'w', encoding='utf-8') as f:
f.writelines(''.join(title).strip())
f.write('\n')
f.writelines(createtime)
f.write('\n')
f.writelines(''.join(content).replace('\n', '').strip())
f.close()
# ---------------------------save image-----------------------------
content = tree.xpath('//*[@id="js_content"]//p/img/@data-src')
picbasename = filebasename.replace('data', 'pic')
if not os.path.exists(picbasename):
os.makedirs(picbasename)
for j, piclink in enumerate(content):
with open(picbasename + f"{len(os.listdir(filebasename))}" + "-" + str(j + 1) + ".jpg", 'wb') as f:
f.write(requests.get(piclink).content)
f.close()
# ---------------------------save image over-----------------------------
time.sleep(0.5)
time.sleep(1)
time.sleep(1)
iii += 1
没有合适的资源?快使用搜索试试~ 我知道了~
资源推荐
资源详情
资源评论
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![rar](https://img-home.csdnimg.cn/images/20210720083606.png)
![rar](https://img-home.csdnimg.cn/images/20210720083606.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![rar](https://img-home.csdnimg.cn/images/20210720083606.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![pdf](https://img-home.csdnimg.cn/images/20210720083512.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083646.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![pdf](https://img-home.csdnimg.cn/images/20210720083512.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![rar](https://img-home.csdnimg.cn/images/20210720083606.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![docx](https://img-home.csdnimg.cn/images/20210720083331.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
![zip](https://img-home.csdnimg.cn/images/20210720083736.png)
收起资源包目录
![package](https://csdnimg.cn/release/downloadcmsfe/public/img/package.f3fc750b.png)
![file-type](https://csdnimg.cn/release/download/static_files/pc/images/minetype/UNKNOWN.png)
![file-type](https://csdnimg.cn/release/download/static_files/pc/images/minetype/UNKNOWN.png)
![file-type](https://csdnimg.cn/release/download/static_files/pc/images/minetype/UNKNOWN.png)
![file-type](https://csdnimg.cn/release/download/static_files/pc/images/minetype/UNKNOWN.png)
共 4 条
- 1
资源评论
![avatar-default](https://csdnimg.cn/release/downloadcmsfe/public/img/lazyLogo2.1882d7f4.png)
![avatar](https://profile-avatar.csdnimg.cn/3a960eebfb74436387ab53fa69fa7a7e_nanfang122.jpg!1)
南方382
- 粉丝: 18
- 资源: 1
上传资源 快速赚钱
我的内容管理 展开
我的资源 快来上传第一个资源
我的收益
登录查看自己的收益我的积分 登录查看自己的积分
我的C币 登录后查看C币余额
我的收藏
我的下载
下载帮助
![voice](https://csdnimg.cn/release/downloadcmsfe/public/img/voice.245cc511.png)
![center-task](https://csdnimg.cn/release/downloadcmsfe/public/img/center-task.c2eda91a.png)
最新资源
资源上传下载、课程学习等过程中有任何疑问或建议,欢迎提出宝贵意见哦~我们会及时处理!
点击此处反馈
![feedback](https://img-home.csdnimg.cn/images/20220527035711.png)
![feedback](https://img-home.csdnimg.cn/images/20220527035711.png)
![feedback-tip](https://img-home.csdnimg.cn/images/20220527035111.png)
安全验证
文档复制为VIP权益,开通VIP直接复制
![dialog-icon](https://csdnimg.cn/release/downloadcmsfe/public/img/green-success.6a4acb44.png)