supesite 自动采集机器人

<?php /* [SupeSite] (C) 2007-2009 Comsenz Inc. $Id: admin_robots.php 11192 2009-02-25 01:45:53Z zhaofei $ */ //外部访问验证 if(!defined('IN_SUPESITE_ADMINCP')) { exit('Access Denied'); } //权限 /* if(!checkperm('managerobots')) { showmessage('no_authority_management_operation'); } */ include_once(S_ROOT.'./function/robot.func.php'); $perpage = 20; //计算起始记录 empty($_GET['page']) ? $page = 1 : $page = intval($_GET['page']); ($page < 1) ? $page=1:''; $start = ($page - 1) * $perpage; //对于页面标签的点亮样式附件 $addclass = $extclass = $viewclass = $importclass = ''; if (empty($_GET['op'])) { $viewclass = ' class="active"'; } elseif ($_GET['op'] == 'add') { $addclass = ' class="active"'; } elseif ($_GET['op'] == 'add') { $extclass = ' class="active"'; } elseif ($_GET['op'] == 'import') { $importclass = ' class="active"'; } $listarr = array(); $thevalue = array(); $importvalue = array(); //对页面提交的处理操作 if(submitcheck('importsubmit')) { //导入采集器的表单提交处理 $importdata = preg_replace("/(#.*\s+)*/", '', $_POST['importtext']); //替换采集器中的注释问份 @$thevalue = unserialize(base64_decode($importdata)); //对采集器编码时行base64解码处理并进行反序列化操作转为可用的数组变量 //反序列化后，如果结果不是数组，或版本号为空，则报出”机器人配置信息不正确“ if(!is_array($thevalue) || empty($thevalue['version'])) { showmessage('robot_import_data_invalid'); } //对不同版本的采集机器为验证 if(empty($_POST['ignoreversion']) && strip_tags($thevalue['version']) != strip_tags(S_VER)) { showmessage('robot_import_version_invalid'); } //采集器名称为空，则对将当前的时间戳做为采集器文件名 if(empty($thevalue['name'])) $thevalue['name'] = $_SGLOBAL['timestamp']; unset($thevalue['robotid'], $thevalue['version']); //销毁采集器记录的ID与版本号 $thevalue = saddslashes($thevalue); //对值重新addslashes操作 $insertsqlarr = $thevalue; $insertsqlarr['uid'] = $_SGLOBAL['supe_uid']; $insertsqlarr['dateline'] = $_SGLOBAL['timestamp']; $robotid = inserttable('robots', $insertsqlarr, 1); //将导入的采集器写入数据库 updaterobot($robotid); //更新采集器缓存 showmessage('robot_import_success', $theurl); } elseif (submitcheck('valuesubmit')) { //采集器编辑与添加的提交表单的处理 $postlisturl = addslashes(serialize(array('manual'=>$_POST['listurl_manual'], 'auto'=>$_POST['listurl_auto']))); $_POST['autotype'] = !empty($_POST['autotype']) && intval($_POST['autotype']) == 2 ? 2 : 1; if(empty($_POST['name'])) $_POST['name'] = $_SGLOBAL['timestamp']; $_POST['subjectreplace'] = !empty($_POST['subjectreplace']) ? implode("\n", $_POST['subjectreplace']) : ''; $_POST['subjectreplaceto'] = !empty($_POST['subjectreplaceto']) ? implode("\n", $_POST['subjectreplaceto']) : ''; $_POST['messagereplace'] = !empty($_POST['messagereplace']) ? implode("\n", $_POST['messagereplace']) : ''; $_POST['messagereplaceto'] = !empty($_POST['messagereplaceto']) ? implode("\n", $_POST['messagereplaceto']) : ''; $setsqlarr = array( 'name' => $_POST['name'], 'dateline' => $_SGLOBAL['timestamp'], 'listurltype'=> 'new', 'listurl' => $postlisturl, 'listpagestart' => $_POST['listpagestart'], 'listpageend' => $_POST['listpageend'], 'allnum' => $_POST['allnum'], 'pernum' => $_POST['pernum'], 'importcatid' => intval($_POST['importcatid']), 'reverseorder' => intval($_POST['reverseorder']), 'encode' => $_POST['encode'], 'savepic' => $_POST['savepic'], 'saveflash' => $_POST['saveflash'], 'subjecturlrule' => striptbr($_POST['subjecturlrule']), 'subjecturllinkrule' => striptbr($_POST['subjecturllinkrule']), 'subjecturllinkpre' => $_POST['subjecturllinkpre'], 'subjectrule' => striptbr($_POST['subjectrule']), 'subjectfilter' => striptbr($_POST['subjectfilter']), 'subjectreplace' => $_POST['subjectreplace'], 'subjectreplaceto' => $_POST['subjectreplaceto'], 'subjectkey' => $_POST['subjectkey'], 'subjectallowrepeat' => $_POST['subjectallowrepeat'], 'datelinerule' => striptbr($_POST['datelinerule']), 'fromrule' => striptbr($_POST['fromrule']), 'authorrule' => striptbr($_POST['authorrule']), 'messagerule' => striptbr($_POST['messagerule']), 'messagefilter' => striptbr($_POST['messagefilter']), 'messagepagetype' => $_POST['messagepagetype'], 'messagepagerule' => striptbr($_POST['messagepagerule']), 'messagepageurlrule' => striptbr($_POST['messagepageurlrule']), 'messagepageurllinkpre' => $_POST['messagepageurllinkpre'], 'messagereplace' => $_POST['messagereplace'], 'messagereplaceto' => $_POST['messagereplaceto'], 'picurllinkpre' => $_POST['picurllinkpre'], 'autotype' => $_POST['autotype'], 'wildcardlen' => $_POST['autotype'] == 1 ? $_POST['wildcardlen'] : '', 'subjecturllinkcancel' => striptbr($_POST['subjecturllinkcancel']), 'subjecturllinkfilter' => striptbr($_POST['subjecturllinkfilter']), 'subjecturllinkpf' => $_POST['subjecturllinkpf'], 'subjectkeycancel' => $_POST['subjectkeycancel'], 'messagekey' => $_POST['messagekey'], 'messagekeycancel' => $_POST['messagekeycancel'], 'messageformat' => $_POST['messageformat'], 'messagepageurllinkpf' => $_POST['messagepageurllinkpf'], 'uidrule' => shtmlspecialchars($_POST['uidrule']), 'defaultdateline' => empty($_POST['defaultdateline']) ? 0 : sstrtotime($_POST['defaultdateline']) ); //对于新增的采集器与编辑的采集器的分别处理 if(empty($_POST['robotid'])) { $robotid = 0; $setsqlarr['uid'] = $_SGLOBAL['supe_uid']; $robotid = inserttable('robots', $setsqlarr, 1); updaterobot($robotid); //更新采集器缓存 showmessage('robot_add_success', $theurl); } else { //UPDATE $wheresqlarr = array( 'robotid' => $_POST['robotid'] ); updatetable('robots', $setsqlarr, $wheresqlarr); updaterobot($_POST['robotid']); //更新采集器缓存 showmessage('robot_edit_success', $theurl); } } elseif (submitcheck('debug')) { //采集器编辑调试用 @ini_set('max_execution_time', 2000); //设置超时时间 $_POST['debugprocess'] = !empty($_POST['debugprocess']) ? trim($_POST['debugprocess']) : 0; if(empty($_POST['debugprocess'])) { showprogress($alang['robot_debug_no_process'], 1); exit(); } //初始化 $_POST['listurl_manual'] = !empty($_POST['listurl_manual']) && is_array($_POST['listurl_manual']) ? $_POST['listurl_manual'] : array(); $_POST['debugurl'] = !empty($_POST['debugurl']) ? trim($_POST['debugurl']) : ''; //start $listurlarr = $listurlarr2 = $infoarr = array(); //初始采集页面数组 $output = ''; $sourcehtml = ''; $sourcecharset = ''; $rule = ''; $i = $urlorder = 0; //对采集数组进行整理 if(empty($_POST['debugurl'])) { if(!empty($_POST['listurl_auto'])) { $_POST['autotype'] = !empty($_POST['autotype']) && intval($_POST['autotype']) == 2 ? 2 : 1; $_POST['listpagestart'] = !empty($_POST['autotype']) && $_POST['autotype'] == 1? intval($_POST['listpagestart']) : ord($_POST['listpagestart']); $_POST['listpageend'] = !empty($_POST['autotype']) && $_POST['autotype'] == 1? intval($_POST['listpageend']) : ord($_POST['listpageend']); $_POST['wildcardlen'] = !empty($_POST['wildcardlen']) ? intval($_POST['wildcardlen']) : 0; if($_POST['listpagestart'] > $_POST['listpageend']) { $urlorder = $_POST['listpagestart']; $_POST['listpagestart'] = $_POST['listpageend']; $_POST['listpageend'] = $urlorder; $urlorder = 1; } for($i = $_POST['listpagestart']; $i <= $_POST['listpageend']; $i++) { $strreplace = $i; if(!empty($_POST['wildcardlen']) && $_POST['autotype'] == 1) { $strreplace = str_pad($i, $_POST['wildcardlen'], 0, STR_PAD_LEFT); } elseif($_POST['autotype'] == 2) { $strreplace = chr($i); } if($_POST['autotype'] == 1 || ($_POST['autotype'] == 2 && preg_match("/[a-z]/i", $strreplace))) { $listurlarr2[] = preg_replace("/\[page\]/", $strreplace, $_POST['listurl_auto']); } } if($