瑞数5代（rs5）补环境+后缀算法（含python爬虫与JS算法）

""" 要想用另一种方式去执行js，而不是使用execjs， 我们可以使用subprocess.run 去执行 node -e "console.log(require('./web.js').myFun(params))" 也可以先node 再 var web = require('./web.js'); web.myFun(video_urls) 即打印最终结果 两种方式的前提条件是：本地环境有node，且需要在js文件中加入 module.exports.myFun = decrypt_m3u8; 封装了平时对execjs的使用 """ import os # from node_vm2 import VM, NodeVM import re import execjs # pip install PyExecJS class ExecjsUtils: def __init__(self, js_file_path: str, cwd=None): self.__choice_execjs_runtime() with open(js_file_path, 'r', encoding='utf-8') as f: print(execjs.get().name) # Node.js (V8) node = execjs.get() # 安装nodejs后，会得到运行环境名为：Node.js (V8) self.__ctx = node.compile(f.read()) def __choice_execjs_runtime(self): os.environ['EXECJS_RUNTIME'] = "Node" def call(self, *args, **kwargs): result = self.__ctx.call(*args, **kwargs) return result import requests from lxml import etree from urllib.parse import urlencode from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) requests = requests.session() class Spider: headers = { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "Accept-Language": "zh-CN,zh;q=0.9", "Cache-Control": "no-cache", "Connection": "keep-alive", "Pragma": "no-cache", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36", "sec-ch-ua": "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\"Windows\"" } head_url = 'https://www.nmpa.gov.cn/' def __init__(self): pass def first_req_header(self): response = requests.get(self.head_url, headers=self.headers, verify=False) print("第一次响应状态；",response.status_code) meta_content = re.search(r'<meta content="(.*?)">', response.text).group(1) js_code = re.search(r'r="m">(.*?)</s', response.text).group(1) print("re匹配 meta_content:", meta_content) print("re匹配自执行js_code:", js_code) end_code = f""" Meta$content = "{meta_content}"; {js_code};; function get_cookie(){{ return document.cookie; }} ;; globalThis.get_cookie = get_cookie; function get_mme(){{ XMLHttpRequest.prototype.open("GET","http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin",true); return XMLHttpRequest.prototype.uri; }} globalThis.get_mme = get_mme; """ return end_code def get_js_utils(self,end_code): head_file = 'rs5_debugger.js' # head_file = 'rs_debugger_fina.js' js_file_path = 'rs5_debugger_dev.js' with open(head_file, 'r', encoding='utf-8') as r, open(js_file_path, 'w', encoding='utf-8') as w: head_code = r.read() w.write(head_code + end_code) utils = ExecjsUtils(js_file_path) return utils def second_req_header(self,end_code:str): self.utils = self.get_js_utils(end_code) cookie_str: str = self.utils.call("get_cookie") cookie_k, cookie_v = cookie_str.split(";", 1)[-1].split('=', 1) cookie_k = cookie_k.strip() cookie_v = cookie_v.strip() print("最终cookie_t:", len(cookie_v),cookie_k, cookie_v) requests.cookies.set(cookie_k, cookie_v) self.headers.update( {"Referer": "https://www.nmpa.gov.cn/","Sec-Fetch-Site": "same-origin"}) response = requests.get(self.head_url, headers=self.headers, verify=False) response.encoding = 'utf-8' print(f"第二次响应状态：{response.status_code}") print(f"第二次响应内容：{response.text}") def mm_req(self): # js_file_path = 'rs_debugger_dev.js' # js_file_path = 'rs_debugger_dev.js' # utils = ExecjsUtils(js_file_path) mme: str = self.utils.call("get_mme") mme = mme.strip("http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin?MmEwMD=") print("MmEwMD:",len(mme),mme) simple_url = 'http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin' params = { "MmEwMD": mme } _url = simple_url + f'?{urlencode(params)}' headers = { "Host": "www.fangdi.com.cn", "Pragma": "no-cache", "Cache-Control": "no-cache", "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", "Origin": "http://www.fangdi.com.cn", "Referer": "http://www.fangdi.com.cn/new_house/new_house_detail.html?project_id=b87cb3d04f878ca6", "Accept-Language": "zh-CN,zh;q=0.9", } _data = { "projectID": "b87cb3d04f878ca6" } data = urlencode(_data) response = requests.post(_url, headers=headers, data=data, verify=False) print(response.status_code,response.text) def main(self): end_code = self.first_req_header() self.second_req_header(end_code) # self.mm_req() # self.mm_req() if __name__ == '__main__': Spider().main() ''' mme: 355 http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin?MmEwMD=4nh651SnrwUUH9smp9lvSFmFjFs62QChZMeWxc4EAaphPieUuTRPA5pNa0.yRQD5u5yqdmwHptNGrqnc0qy3vw.ENIdddd6ddddedddddIdddduuXbiSXZ8Mwg3QBxRgozu8.Jl4FYqbXmi.Oel_sUT62knBgof3G5K1Nhv4nj6yopFyCYNQdnbVIgRTTBkrQJfojCuaaqPes1s5ss8NJzl.IIdddd6ddddedddddIddddeB_ZjsA8DeGJ7qOG41oD_5wiKodc9SCcxk1jgs8Kvfv mme: 355 http://www.fangdi.com.cn/service/freshHouse/queryProjectById.actin?MmEwMD=4nh651SnrwUUH9smp9lvSFmFjFs62QChZMeWxc4EAaphPieUuTRPA5pNa0.yRQD5u5yqdmwHptNGrqnc0qy3vw.ENIdddd6ddddedddddIdddduuXbiSXZ8Mwg3QBxRgozu8.Jl4FYqbXmi.Oel_sUT62knBgof3G5K1Nhv4nj6yopFyCYNQdnbVIgRTTBkrQJfojCuaaqPes1s5ss8NJzl.IIdddd6ddddedddddIddddeB_ZjsA8DeGJ7qOG41oD_5wiKodc9SCcxk1jgs8Kvfv '''