jinql
/
skycaiji
mirror of https://gitee.com/zorlan/skycaiji
1
0
Fork 0
Code Issues Releases Wiki Activity

2.1

pull/1/head
zorlan 2019-03-14 17:13:21 +08:00
parent fef736b6bb
commit 8388999a1b
64 changed files with 1321 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
SkycaijiApp/admin/behavior/Init.php
View File

@ -30,7 +30,7 @@ class Init{
/*自动登录*/ /*自动登录*/
$muser=model('User'); $muser=model('User');
$s_userid=session('user_id'); $s_userid=session('user_id');
if(empty($s_userid)){ if(!IS_CLI&&empty($s_userid)){
$login_history=cookie('login_history'); $login_history=cookie('login_history');
if(!empty($login_history)){ if(!empty($login_history)){
@ -95,6 +95,18 @@ class Init{
$configList=$cacheConfig['list']; $configList=$cacheConfig['list'];
} }
$GLOBALS['config']=$configList; $GLOBALS['config']=$configList;
if(!empty($GLOBALS['config']['site']['closelog'])){
\think\Log::init(array('type'=>'test','level'=>array()));
}
if(!empty($GLOBALS['config']['site']['dblong'])){
$dbParams=config('database.params');
$dbParams[\PDO::ATTR_PERSISTENT]=true;
config('database.params',$dbParams);
}
$GLOBALS['clientinfo']=clientinfo(); $GLOBALS['clientinfo']=clientinfo();
if(!empty($GLOBALS['clientinfo'])){ if(!empty($GLOBALS['clientinfo'])){
$GLOBALS['clientinfo']=base64_encode(json_encode($GLOBALS['clientinfo'])); $GLOBALS['clientinfo']=base64_encode(json_encode($GLOBALS['clientinfo']));

87
SkycaijiApp/admin/command/Collect.php Normal file
View File

@ -0,0 +1,87 @@
<?php
/*
|--------------------------------------------------------------------------
| SkyCaiji (蓝天采集器)
|--------------------------------------------------------------------------
| Copyright (c) 2018 http://www.skycaiji.com All rights reserved.
|--------------------------------------------------------------------------
| 使用协议 http://www.skycaiji.com/licenses
|--------------------------------------------------------------------------
*/
namespace skycaiji\admin\command;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\input\Option;
use think\console\Output;
use skycaiji\admin\model\CacheModel;
class Collect extends Command{
protected function configure(){
$this->setName('collect')
->addArgument('op', Argument::OPTIONAL, "op")
->addOption('cli_user', null, Option::VALUE_REQUIRED, 'cli user')
->addOption('task_id', null, Option::VALUE_REQUIRED, 'collect:task_id')
->addOption('task_ids', null, Option::VALUE_REQUIRED, 'batch:task_ids')
->setDescription('collect task');
}
protected function execute(Input $input, Output $output){
$cacheConfig=CacheModel::getInstance()->getCache('cli_cache_config','data');
if(is_array($cacheConfig)){
\think\Config::set($cacheConfig);
}
if ($input->hasOption('cli_user')){
$cliUser=$input->getOption('cli_user');
$cliUser=base64_decode($cliUser);
$cliUser=explode('_', $cliUser);
$muser=new \skycaiji\admin\model\User();
$user=$muser->where('username',$cliUser[0])->find();
if(!empty($user)){
$user['username']=strtolower($user['username']);
if($user['username']==$cliUser[0]&&$cliUser[1]==md5($user['username'].$user['password'])){
session('user_id',$user['uid']);
}
}
}
$op=$input->getArgument('op');
if('task'==$op){
$taskId=0;
if ($input->hasOption('task_id')){
$taskId=$input->getOption('task_id');
$taskId=intval($taskId);
}
$curUrl=\think\Config::get('root_website').'/admin/task/collect?backstage=1&id='.urlencode($taskId);
\think\Request::create($curUrl);
define('BIND_MODULE', "admin/task/collect");
}elseif('auto'==$op){
$curUrl=\think\Config::get('root_website').'/admin/api/collect?backstage=1';
\think\Request::create($curUrl);
define('BIND_MODULE', "admin/api/collect");
}elseif('batch'==$op){
$taskIds='';
if ($input->hasOption('task_ids')){
$taskIds=$input->getOption('task_ids');
}
$curUrl=\think\Config::get('root_website').'/admin/task/collectBatch?backstage=1&ids='.urlencode($taskIds);
\think\Request::create($curUrl);
define('BIND_MODULE', "admin/task/collectBatch");
}
\think\App::run()->send();
}
}

60
SkycaijiApp/admin/common.php
View File

@ -89,4 +89,64 @@ function is_collecting(){
/*移除自动采集》正在采集状态*/ /*移除自动采集》正在采集状态*/
function remove_auto_collecting(){ function remove_auto_collecting(){
\skycaiji\admin\model\CacheModel::getInstance()->db()->where('cname','auto_collecting')->delete(); \skycaiji\admin\model\CacheModel::getInstance()->db()->where('cname','auto_collecting')->delete();
}
/*cli命令行*/
function cli_command_exec($paramStr){
if(config('cli_cache_config')){
$cacheConfig=\skycaiji\admin\model\CacheModel::getInstance()->getCache('cli_cache_config','data');
$cliConfig=array();
foreach (config('cli_cache_config') as $key){
$cliConfig[$key]=config($key);
}
if(serialize($cacheConfig)!=serialize($cliConfig)){
\skycaiji\admin\model\CacheModel::getInstance()->setCache('cli_cache_config',$cliConfig);
}
}
static $php_ext_path=null;
if(!isset($php_ext_path)){
$ini_all=ini_get_all();
$php_ext_path=$ini_all['extension_dir']['local_value'];
}
if($php_ext_path){
$phpPath=preg_replace('/[\/\\\]ext[\/\\\]*$/i', '', $php_ext_path);
if(IS_WIN){
$phpPath.=DIRECTORY_SEPARATOR.'php.exe';
}
}else{
$phpPath='php';
}
$commandStr=$phpPath;
if(IS_WIN){
$commandStr='"'.$commandStr.'"';
}
$cliUser=strtolower($GLOBALS['user']['username']);
$cliUser=$cliUser.'_'.md5($cliUser.$GLOBALS['user']['password']);
$paramStr.=' --cli_user '.base64_encode($cliUser);
$commandStr.=' '.config('root_path').DIRECTORY_SEPARATOR.'caiji '.$paramStr;
$descriptorspec = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('pipe', 'w')
);
$pipes=array();
$handle=proc_open($commandStr,$descriptorspec,$pipes);
$hdStatus=proc_get_status($handle);
fclose($pipes[0]);
fclose($pipes[1]);
fclose($pipes[2]);
exit();
} }

14
SkycaijiApp/admin/controller/Api.php
View File

@ -28,6 +28,18 @@ class Api extends BaseController{
} }
/*执行采集*/ /*执行采集*/
public function collectAction(){ public function collectAction(){
if(input('?backstage')){
if(!IS_CLI){
ignore_user_abort(true);
if($GLOBALS['config']['caiji']['server']=='cli'){
cli_command_exec('collect auto');
exit();
}
}
}
define('IS_COLLECTING', 1); define('IS_COLLECTING', 1);
$mcache=CacheModel::getInstance(); $mcache=CacheModel::getInstance();
if($mcache->getCache('auto_collecting')){ if($mcache->getCache('auto_collecting')){
@ -37,7 +49,7 @@ class Api extends BaseController{
$mcache->setCache('auto_collecting',1); $mcache->setCache('auto_collecting',1);
register_shutdown_function('remove_auto_collecting'); register_shutdown_function('remove_auto_collecting');
if(!session('user_id')){ if(input('?backstage')||!session('user_id')){
define('CLOSE_ECHO_MSG', true); define('CLOSE_ECHO_MSG', true);
} }

111
SkycaijiApp/admin/controller/Backstage.php
View File

@ -110,6 +110,117 @@ class Backstage extends BaseController{
header('Content-Type:application/json;charset=utf-8'); header('Content-Type:application/json;charset=utf-8');
exit($html); exit($html);
} }
/*后台任务操作*/
public function backstageTaskAction(){
$op=input('op');
$mcache=CacheModel::getInstance('backstage_task');
if(empty($op)){
$count0=$mcache->db()->where('ctype',0)->count();
$count1=$mcache->db()->where('ctype',1)->count();
$this->assign('count0',$count0);
$this->assign('count1',$count1);
return $this->fetch('bk_task');
}elseif('count'==$op){
$count=$mcache->db()->where('ctype',0)->count();
$count=intval($count);
$this->success('','',array('count'=>$count));
}elseif('task0'==$op||'task1'==$op){
$taskType=('task0'==$op)?0:1;
$list=$mcache->db()->where('ctype',$taskType)->order('dateline desc')->paginate(10,false,paginate_auto_config());
$pagenav=$list->render();
$list=$list->all();
$cacheList=array();
if($list){
foreach ($list as $k=>$v){
$v['cname']=intval($v['cname']);
if($taskType){
$v['endtime']=intval($v['data']);
$v['enddate']=date('Y-m-d H:i:s',$v['endtime']);
}
$cacheList[$v['cname']]=$v;
}
$list=model('Task')->where('id','in',array_keys($cacheList))->column('*','id');
$nullIds=array();
$list1=array();
foreach ($cacheList as $k=>$v){
if(!isset($list[$k])){
$nullIds[$k]=$k;
}else{
$list1[$k]=$list[$k];
}
}
$list=$list1;
if(!empty($nullIds)&&is_array($nullIds)){
$mcache->db()->where('cname','in',$nullIds)->delete();
}
if($taskType&&is_array($cacheList)){
foreach ($cacheList as $k=>$v){
$cond=array(
'task_id'=>$k,
'addtime'=>array('between',array($v['dateline'],$v['endtime']))
);
$cacheList[$k]['collected_count']=model('Collected')->where($cond)->count();
}
}
}
$count=$mcache->db()->where('ctype',$taskType)->count();
$this->assign('list',$list);
$this->assign('cacheList',$cacheList);
$this->assign('taskType',$taskType);
$this->assign('pagenav',$pagenav);
$html=$this->fetch('bk_task_list')->getContent();
$this->success('',null,array('html'=>$html,'count'=>$count));
}elseif('collected'==$op){
$taskId=input('tid/d');
if($taskId<=0){
$this->error('任务id错误');
}
$cache=$mcache->db()->where('cname',$taskId)->find();
if(empty($cache)){
$this->error('后台任务不存在');
}
$cond=array('task_id'=>$taskId);
$taskStatus=$cache['ctype'];
if(empty($taskStatus)){
$cond['addtime']=array('>=',$cache['dateline']);
}else{
$cond['addtime']=array('between',array($cache['dateline'],intval($cache['data'])));
}
$list=model('Collected')->where($cond)->order('addtime desc')->paginate(10,false,paginate_auto_config());
$pagenav=$list->render();
$list=$list->all();
$this->assign('list',$list);
$this->assign('pagenav',$pagenav);
$this->assign('taskStatus',$taskStatus);
$this->assign('taskId',$taskId);
return $this->fetch('bk_task_collected');
}
}
/*生成js语言包文件*/ /*生成js语言包文件*/
public function createJsLangAction(){ public function createJsLangAction(){
$langs=array(); $langs=array();

6
SkycaijiApp/admin/controller/BaseController.php
View File

@ -67,8 +67,10 @@ class BaseController extends \skycaiji\common\controller\BaseController{
} }
} }
echo '<p style="color:'.$color.';">'.$str.'</p>'.$end_str; echo '<p style="color:'.$color.';">'.$str.'</p>'.$end_str;
ob_flush(); if(ob_get_level()>0){
flush(); ob_flush();
flush();
}
} }
} }
/*保留旧的入口*/ /*保留旧的入口*/

15
SkycaijiApp/admin/controller/Collected.php
View File

@ -50,13 +50,14 @@ class Collected extends BaseController {
$cond['error']=array('<>',''); $cond['error']=array('<>','');
} }
} }
$dataList=array();
$taskList=array();
if(!$null_task){ if(!$null_task){
$count=$mcollected->where($cond)->count(); $count=$mcollected->where($cond)->count();
$limit=$search['num']; $limit=$search['num'];
if($count>0){ if($count>0){
$dataList=$mcollected->where($cond)->order('id desc')->paginate($limit,false,array('query'=>$search)); $dataList=$mcollected->where($cond)->order('id desc')->paginate($limit,false,paginate_auto_config());
$pagenav=$dataList->render(); $pagenav=$dataList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);
@ -74,16 +75,20 @@ class Collected extends BaseController {
if(!empty($taskIds)){ if(!empty($taskIds)){
$taskList=model('Task')->where(array('id'=>array('in',$taskIds)))->column('name','id'); $taskList=model('Task')->where(array('id'=>array('in',$taskIds)))->column('name','id');
} }
$this->assign('dataList',$dataList);
$this->assign('taskList',$taskList);
} }
$GLOBALS['content_header']=lang('collected_list'); $GLOBALS['content_header']=lang('collected_list');
$GLOBALS['breadcrumb']=breadcrumb(array(array('url'=>url('Collected/list'),'title'=>lang('collected_list')))); $GLOBALS['breadcrumb']=breadcrumb(array(array('url'=>url('Collected/list'),'title'=>lang('collected_list'))));
} }
$this->assign('search',$search); $this->assign('search',$search);
$this->assign('dataList',$dataList);
$this->assign('taskList',$taskList);
return $this->fetch(); return $this->fetch();
} }
/*清理失败的数据*/
public function clearErrorAction(){
model('Collected')->where("`error` is not null and `error`<>''")->delete();
$this->success('清理完成','Admin/Collected/list');
}
/** /**
* 操作 * 操作
*/ */

7
SkycaijiApp/admin/controller/Collector.php
View File

@ -79,19 +79,16 @@ class Collector extends BaseController {
public function listAction(){ public function listAction(){
$page=max(1,input('p/d',0)); $page=max(1,input('p/d',0));
$module=input('module'); $module=input('module');
$pageParams=array();
$cond=array(); $cond=array();
$taskCond=array(); $taskCond=array();
if(!empty($module)){ if(!empty($module)){
$cond=array('module'=>$module); $cond=array('module'=>$module);
$pageParams['module']=$module;
} }
$mcoll=model('Collector'); $mcoll=model('Collector');
$limit=20; $limit=20;
$count=$mcoll->where($cond)->count(); $count=$mcoll->where($cond)->count();
$collList=$mcoll->where($cond)->paginate($limit,false,array('query'=>$pageParams)); $collList=$mcoll->where($cond)->paginate($limit,false,paginate_auto_config());
$pagenav = $collList->render(); $pagenav = $collList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);

118
SkycaijiApp/admin/controller/Cpattern.php
View File

@ -94,11 +94,16 @@ class Cpattern extends BaseController {
}else{ }else{
$this->error('请输入正确的网址'); $this->error('请输入正确的网址');
} }
}elseif($source['type']=='api'){
if(!preg_match('/^\w+\:\/\//i',$source['api'])){
$this->error('请输入正确的api网址');
}
$urlFmt=$source['api'].'{json:'.$source['api_json'].'}';
} }
if($urls){ if($urls||$urlFmt){
$urls=array_values($urls); $urls=$urls?array_values($urls):'';
$this->success('',null,array('uid'=>$source['uid'],'url'=>$urlFmt,'urls'=>$urls)); $this->success('',null,array('uid'=>$source['uid'],'url'=>$urlFmt,'urls'=>$urls));
}else{ }else{
$this->error('未生成网址！'); $this->error('未生成网址！');
@ -109,7 +114,7 @@ class Cpattern extends BaseController {
if($url){ if($url){
$source['uid']=input('uid',''); $source['uid']=input('uid','');
if(preg_match('/\{param\:(\w+)\,([^\}]*)\}/i', $url,$param)){ if(preg_match('/\{param\:(\w+)\,([^\}]*)\}/i',$url,$param)){
$source['url']= preg_replace('/\{param\:(\w+)\,([^\}]*)\}/i', cp_sign('match'), $url); $source['url']= preg_replace('/\{param\:(\w+)\,([^\}]*)\}/i', cp_sign('match'), $url);
$source['type']='batch'; $source['type']='batch';
@ -128,6 +133,11 @@ class Cpattern extends BaseController {
$source['param_custom']=implode("\r\n", $param_val); $source['param_custom']=implode("\r\n", $param_val);
} }
}elseif(preg_match('/\{json\:([^\}]*)\}/i',$url,$json)){
$source['type']='api';
$source['api']=preg_replace('/\{json\:([^\}]*)\}/i','',$url);
$source['api_json']=$json[1];
}elseif(preg_match('/[\r\n]/', $url)){ }elseif(preg_match('/[\r\n]/', $url)){
$source['type']='large'; $source['type']='large';
@ -339,24 +349,32 @@ class Cpattern extends BaseController {
$taskData=model('Task')->getById($eCpattern->collector['task_id']); $taskData=model('Task')->getById($eCpattern->collector['task_id']);
model('Task')->loadConfig($taskData['config']); model('Task')->loadConfig($taskData['config']);
$GLOBALS['breadcrumb']=breadcrumb(array(array('url'=>url('Collector/set?task_id='.$taskData['id']),'title'=>lang('task').lang('separator').$taskData['name']),'测试'));
if('source_urls'==$op){ if('source_urls'==$op){
$source_urls=array(); $source_urls=array();
foreach ( $eCpattern->config ['source_url'] as $k => $v ) { foreach ($eCpattern->config['source_url'] as $k => $v) {
if(empty($v)){ if(empty($v)){
continue; continue;
} }
$urls = $eCpattern->convert_source_url ( $v ); $source_urls[$v] = $eCpattern->convert_source_url ( $v );
if (is_array ( $urls )) { }
$source_urls = array_merge ( $source_urls, $urls ); if(!$eCpattern->config['source_is_url']){
} else {
$source_urls [] = $urls; $source_urls1=array();
foreach ($source_urls as $k=>$v){
if (is_array ( $v )) {
$source_urls1 = array_merge ( $source_urls1, $v );
} else {
$source_urls1 [] = $v;
}
} }
$source_urls=$source_urls1;
} }
$eCpattern->assign('source_urls',$source_urls); $eCpattern->assign('source_urls',$source_urls);
$eCpattern->assign('config',$eCpattern->config); $eCpattern->assign('config',$eCpattern->config);
return $eCpattern->fetch('cpattern:test_source_urls'); return $eCpattern->fetch('cpattern:test_source_urls');
}elseif('cont_urls'==$op){ }elseif('cont_urls'==$op){
@ -365,19 +383,38 @@ class Cpattern extends BaseController {
$curLevel=$curLevel>0?$curLevel:0; $curLevel=$curLevel>0?$curLevel:0;
$levelData=$eCpattern->get_level_urls($source_url,$curLevel); $levelData=$eCpattern->get_level_urls($source_url,$curLevel);
$eCpattern->success('',null,array('urls'=>$levelData['urls'],'levelName'=>$levelData['levelName'],'nextLevel'=>$levelData['nextLevel'])); $eCpattern->success('',null,array('urls'=>$levelData['urls'],'levelName'=>$levelData['levelName'],'nextLevel'=>$levelData['nextLevel']));
}elseif('cont_url'==$op){ }elseif('cont_url'==$op){
$GLOBALS['content_header']='测试抓取'; $GLOBALS['content_header']='测试抓取';
$GLOBALS['breadcrumb']=breadcrumb('测试抓取');
$cont_url=input('cont_url','','trim'); $cont_url=input('cont_url','','trim');
$test=input('test'); $test=input('test');
$url_post=$eCpattern->config['url_post']; $url_post=$eCpattern->config['url_post'];
$input_urls=array();
foreach ($eCpattern->config['new_field_list'] as $field){
if('source_url'==strtolower($field['field']['source'])){
$input_urls['source_url']=input('source_url');
$input_urls['source_url']=$input_urls['source_url']?$input_urls['source_url']:'';
}elseif(preg_match('/level_url:/i', $field['field']['source'])){
foreach($eCpattern->config['level_urls'] as $levIx=>$levVal){
if($field['field']['source']==('level_url:'.$levVal['name'])){
$level=$levIx+1;
$input_urls['level_url'][$level]=array('level'=>$level,'name'=>$levVal['name'],'url'=>input('level_'.$level));
break;
}
}
}
}
$eCpattern->assign('cont_url',$cont_url); $eCpattern->assign('cont_url',$cont_url);
$eCpattern->assign('url_post',$url_post); $eCpattern->assign('url_post',$url_post);
$eCpattern->assign('input_urls',$input_urls);
$eCpattern->assign('test',$test); $eCpattern->assign('test',$test);
if(request()->isAjax()){ if(request()->isAjax()){
return view('cpattern:test_cont_url_ajax'); return view('cpattern:test_cont_url_ajax');
@ -393,13 +430,27 @@ class Cpattern extends BaseController {
} }
$html='get_fields'==$op?'':$eCpattern->get_html($cont_url,false,$eCpattern->config['url_post']); $html='get_fields'==$op?'':$eCpattern->get_html($cont_url,false,$eCpattern->config['url_post']);
if('get_fields'==$op){ if('get_fields'==$op){
if(input('?source_url')){
$eCpattern->cur_source_url=input('source_url');
}
foreach (input('param.') as $k=>$v){
if(preg_match('/^level_(\d+)$/',$k,$mLevel)){
$mLevel=intval($mLevel[1])-1;
$eCpattern->cur_level_urls[$eCpattern->config['level_urls'][$mLevel]['name']]=$v;
}
}
$val_list=$eCpattern->getFields($cont_url); $val_list=$eCpattern->getFields($cont_url);
if(empty($eCpattern->first_loop_field)){ if(empty($eCpattern->first_loop_field)){
$val_list=array($val_list); $val_list=array($val_list);
} }
foreach ($val_list as $v_k=>$vals){ foreach ($val_list as $v_k=>$vals){
foreach ($vals as $k=>$v){ foreach ($vals as $k=>$v){
$vals[$k]=$v['value']; $vals[$k]=$v['value'];
@ -437,7 +488,6 @@ class Cpattern extends BaseController {
}elseif('match'==$op){ }elseif('match'==$op){
$GLOBALS['content_header']='模拟匹配'; $GLOBALS['content_header']='模拟匹配';
$GLOBALS['breadcrumb']=breadcrumb('模拟匹配');
if(request()->isPost()){ if(request()->isPost()){
$type=strtolower(input('type')); $type=strtolower(input('type'));
$content=input('content','','trim'); $content=input('content','','trim');
@ -476,7 +526,8 @@ class Cpattern extends BaseController {
$val=$eCpattern->field_module_xpath(array('xpath'=>$match,'xpath_attr'=>''), $content); $val=$eCpattern->field_module_xpath(array('xpath'=>$match,'xpath_attr'=>''), $content);
break; break;
case 'json': case 'json':
$val=$eCpattern->field_module_json(array('json'=>$match), $content); $val=$eCpattern->field_module_json(array('json'=>$match,'json_arr_implode'=>"\r\n"), $content);
$val=trim($val);
break; break;
} }
if(empty($val)){ if(empty($val)){
@ -493,8 +544,6 @@ class Cpattern extends BaseController {
} }
}elseif('elements'==$op){ }elseif('elements'==$op){
ob_clean();
header("Content-type:text/html;charset=utf-8");
$cont_url=input('cont_url','','trim'); $cont_url=input('cont_url','','trim');
if(!preg_match('/^\w+\:\/\//',$cont_url)){ if(!preg_match('/^\w+\:\/\//',$cont_url)){
@ -502,8 +551,12 @@ class Cpattern extends BaseController {
} }
$html=$eCpattern->get_html($cont_url,false,$eCpattern->config['url_post']); $html=$eCpattern->get_html($cont_url,false,$eCpattern->config['url_post']);
$html=preg_replace('/<script[^<>]*?>[\s\S]*?<\/script>/i', '', $html); $jsonData=null;
$html=preg_replace('/<meta[^<>]*charset[^<>]*?>/i', '', $html); if(preg_match('/^\{[\s\S]*\}$/',$html)){
$jsonData=json_decode($html,true);
$jsonData=$jsonData?$jsonData:null;
}
$publicUrl=config('root_website').'/public'; $publicUrl=config('root_website').'/public';
@ -511,8 +564,23 @@ class Cpattern extends BaseController {
."\r\n".'<script src="%s/static/js/admin/cpattern_elements.js?%s"></script>' ."\r\n".'<script src="%s/static/js/admin/cpattern_elements.js?%s"></script>'
."\r\n".'<link rel="stylesheet" href="%s/static/css/cpattern_elements.css?%s">'; ."\r\n".'<link rel="stylesheet" href="%s/static/css/cpattern_elements.css?%s">';
$jscss=sprintf($jscss,$publicUrl,config('html_v'),$publicUrl,config('html_v'),$publicUrl,config('html_v')); $jscss=sprintf($jscss,$publicUrl,config('html_v'),$publicUrl,config('html_v'),$publicUrl,config('html_v'));
$html.=$jscss;
exit($html); if(empty($jsonData)){
$html=preg_replace('/<script[^<>]*?>[\s\S]*?<\/script>/i', '', $html);
$html=preg_replace('/<meta[^<>]*charset[^<>]*?>/i', '', $html);
$html.=$jscss."\r\n".'<script>$(document).ready(function(){skycaijiCE.init();});</script>';
ob_clean();
header("Content-type:text/html;charset=utf-8");
exit($html);
}else{
$GLOBALS['content_header']='分析元素';
$eCpattern->assign('html',$html);
$eCpattern->assign('jscss',$jscss);
return $eCpattern->fetch('cpattern:test_elements');
}
} }
} }
/*名称命名规范*/ /*名称命名规范*/

23
SkycaijiApp/admin/controller/Index.php
View File

@ -54,25 +54,17 @@ class Index extends BaseController{
} }
$collectTime1=time(); $collectTime1=time();
try{ try{
@get_html(url('Admin/Api/collect',null,false,true)); @get_html(url('Admin/Api/collect?backstage=1',null,false,true),null,array('timeout'=>3));
}catch(\Exception $ex){ }catch(\Exception $ex){
} }
$collectTime2=time();
$minWaitTime=60; sleep(20);
$waitTime=0;
if($GLOBALS['config']['caiji']['interval']>0){
$waitTime=60*$GLOBALS['config']['caiji']['interval']-($collectTime2-$collectTime1);
}
$waitTime=$waitTime>$minWaitTime?$waitTime:$minWaitTime;
sleep($waitTime);
sleep(10);
if($GLOBALS['config']['caiji']['auto']){ if($GLOBALS['config']['caiji']['auto']){
try{ try{
@get_html(url('Admin/Index/backstage',array('autorun'=>1,'runtime'=>$runtime),false,true),null,array('timeout'=>5)); @get_html(url('Admin/Index/backstage?autorun=1&runtime='.$runtime,null,false,true),null,array('timeout'=>3));
}catch(\Exception $ex){ }catch(\Exception $ex){
} }
@ -82,7 +74,10 @@ class Index extends BaseController{
} }
/*访问执行采集*/ /*访问执行采集*/
public function caijiAction(){ public function caijiAction(){
@get_html(url('Admin/Api/collect',null,false,true),null,array('timeout'=>1)); if(empty($GLOBALS['config']['caiji']['auto'])){
$this->error('请先开启自动采集','Admin/Setting/caiji');
}
@get_html(url('Admin/Api/collect?backstage=1',null,false,true),null,array('timeout'=>3));
$waitTime=$GLOBALS['config']['caiji']['interval']*60; $waitTime=$GLOBALS['config']['caiji']['interval']*60;
$waitTime=$waitTime>0?$waitTime:3; $waitTime=$waitTime>0?$waitTime:3;
$this->success('正在采集...','Admin/Index/caiji',null,$waitTime); $this->success('正在采集...','Admin/Index/caiji',null,$waitTime);
@ -296,7 +291,7 @@ class Index extends BaseController{
$this->error($check['msg']); $this->error($check['msg']);
} }
$muser->save(array('password'=>pwd_encrypt($pwd)),array('username'=>$stepSession['user']['username'])); $muser->strict(false)->where(array('username'=>$stepSession['user']['username']))->update(array('password'=>pwd_encrypt($pwd)));
session($stepSname,null); session($stepSname,null);
$this->success(lang('find_pwd_success'),'Admin/Index/index'); $this->success(lang('find_pwd_success'),'Admin/Index/index');
}else{ }else{

10
SkycaijiApp/admin/controller/Mystore.php
View File

@ -29,13 +29,10 @@ class Mystore extends BaseController {
$type='collect'; $type='collect';
$module=input('module'); $module=input('module');
$page=max(1,input('p/d',0)); $page=max(1,input('p/d',0));
$pageParams=array();
$pageParams['type']=$type;
$cond=array('type'=>$type); $cond=array('type'=>$type);
if(!empty($module)){ if(!empty($module)){
$cond=array('module'=>$module); $cond=array('module'=>$module);
$pageParams['module']=$module;
} }
$sortBy=input('sort','desc'); $sortBy=input('sort','desc');
@ -48,7 +45,7 @@ class Mystore extends BaseController {
$limit=20; $limit=20;
$count=$mrule->where($cond)->count(); $count=$mrule->where($cond)->count();
$ruleList=$mrule->where($cond)->order($orderBy)->paginate($limit,false,array('query'=>$pageParams)); $ruleList = $mrule->where($cond)->order($orderBy)->paginate($limit,false,paginate_auto_config());
$pagenav = $ruleList->render(); $pagenav = $ruleList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);
@ -58,7 +55,7 @@ class Mystore extends BaseController {
$GLOBALS['breadcrumb']=breadcrumb(array(array('url'=>url('Mystore/index'),'title'=>'已下载'),lang('rule_'.$type))); $GLOBALS['breadcrumb']=breadcrumb(array(array('url'=>url('Mystore/index'),'title'=>'已下载'),lang('rule_'.$type)));
$this->assign('ruleList',$ruleList); $this->assign('ruleList',$ruleList);
$tpl=input('tpl'); $tpl=input('tpl');
$tpl='rules'.(!empty($tpl)?('_'.$tpl):''); $tpl='rules'.(!empty($tpl)?('_'.$tpl):'');
@ -120,7 +117,6 @@ class Mystore extends BaseController {
} }
public function releaseAppAction(){ public function releaseAppAction(){
$page=max(1,input('p/d',0)); $page=max(1,input('p/d',0));
$pageParams=array();
$cond=array(); $cond=array();
$sortBy=input('sort','desc'); $sortBy=input('sort','desc');
@ -133,7 +129,7 @@ class Mystore extends BaseController {
$mapp=model('ReleaseApp'); $mapp=model('ReleaseApp');
$limit=20; $limit=20;
$count=$mapp->where($cond)->count(); $count=$mapp->where($cond)->count();
$appList=$mapp->where($cond)->order($orderBy)->paginate($limit,false,array('query'=>$pageParams)); $appList=$mapp->where($cond)->order($orderBy)->paginate($limit,false,paginate_auto_config());
$pagenav = $appList->render(); $pagenav = $appList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);

8
SkycaijiApp/admin/controller/Release.php
View File

@ -47,12 +47,12 @@ class Release extends BaseController{
if(empty($releData)){ if(empty($releData)){
$mrele->allowField(true)->save($newData); $mrele->isUpdate(false)->allowField(true)->save($newData);
$releId=$mrele->id; $releId=$mrele->id;
}else{ }else{
$releId=$releData['id']; $releId=$releData['id'];
$mrele->allowField(true)->save($newData,array('id'=>$releData['id'])); $mrele->strict(false)->where(array('id'=>$releData['id']))->update($newData);
} }
if($releId>0){ if($releId>0){
$this->success(lang('op_success'),'Release/set?task_id='.$taskId); $this->success(lang('op_success'),'Release/set?task_id='.$taskId);
@ -92,11 +92,11 @@ class Release extends BaseController{
$page=max(1,input('p/d',0)); $page=max(1,input('p/d',0));
$mrele=model('Release'); $mrele=model('Release');
$mtask=model('Task'); $mtask=model('Task');
$limit=10; $limit=20;
$cond=array(); $cond=array();
$taskCond=array(); $taskCond=array();
$count=$mrele->where($cond)->count(); $count=$mrele->where($cond)->count();
$releList=$mrele->where($cond)->order('id desc')->paginate($limit); $releList=$mrele->where($cond)->order('id desc')->paginate($limit,false,paginate_auto_config());
$pagenav = $releList->render(); $pagenav = $releList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);

16
SkycaijiApp/admin/controller/Setting.php
View File

@ -19,6 +19,8 @@ class Setting extends BaseController {
$config=array(); $config=array();
$config['verifycode']=input('verifycode/d',0); $config['verifycode']=input('verifycode/d',0);
$config['hidehome']=input('hidehome/d',0); $config['hidehome']=input('hidehome/d',0);
$config['closelog']=input('closelog/d',0);
$config['dblong']=input('dblong/d',0);
$config['login']=input('login/a'); $config['login']=input('login/a');
if($config['login']['limit']){ if($config['login']['limit']){
@ -47,6 +49,7 @@ class Setting extends BaseController {
$config=array(); $config=array();
$config['auto']=input('auto/d',0); $config['auto']=input('auto/d',0);
$config['run']=input('run'); $config['run']=input('run');
$config['server']=input('server');
$config['num']=input('num/d',0); $config['num']=input('num/d',0);
$config['interval']=input('interval/d',0); $config['interval']=input('interval/d',0);
$config['timeout']=input('timeout/d',0); $config['timeout']=input('timeout/d',0);
@ -60,6 +63,12 @@ class Setting extends BaseController {
$config['img_interval']=input('img_interval/d',0); $config['img_interval']=input('img_interval/d',0);
$config['img_max']=input('img_max/d',0); $config['img_max']=input('img_max/d',0);
if($config['server']=='cli'){
if(!function_exists('proc_open')){
$this->error('抱歉cli命令行模式需开启proc_open函数');
}
}
if(!empty($config['img_path'])){ if(!empty($config['img_path'])){
$checkImgPath=$mconfig->check_img_path($config['img_path']); $checkImgPath=$mconfig->check_img_path($config['img_path']);
@ -80,7 +89,7 @@ class Setting extends BaseController {
remove_auto_collecting(); remove_auto_collecting();
if($config['run']=='backstage'){ if($config['run']=='backstage'){
@get_html(url('Admin/Index/backstage',array('autorun'=>1),false,true),null,array('timeout'=>1)); @get_html(url('Admin/Index/backstage?autorun=1',null,false,true),null,array('timeout'=>3));
} }
} }
@ -147,7 +156,7 @@ class Setting extends BaseController {
unset($newData['invalid']); unset($newData['invalid']);
$mproxy->allowField(true)->save($newData,array('ip'=>$newData['ip'])); $mproxy->strict(false)->where(array('ip'=>$newData['ip']))->update($newData);
}else{ }else{
$mproxy->db()->insert($newData,true); $mproxy->db()->insert($newData,true);
@ -283,7 +292,8 @@ class Setting extends BaseController {
$config['tool']=''; $config['tool']='';
} }
if(empty($config['tool'])){ if(empty($config['tool'])){
$this->error('请选择渲染工具','');
} }
if($config['tool']=='chrome'){ if($config['tool']=='chrome'){

5
SkycaijiApp/admin/controller/Store.php
View File

@ -54,10 +54,11 @@ class Store extends BaseController {
$newRule['store_id']=$store_id; $newRule['store_id']=$store_id;
$newRule['addtime']=NOW_TIME; $newRule['addtime']=NOW_TIME;
$ruleId=$mrule->insert($newRule); $mrule->isUpdate(false)->allowField(true)->save($newRule);
$ruleId=$mrule->id;
}else{ }else{
$mrule->allowField(true)->save($newRule,array('id'=>$ruleData['id'])); $mrule->strict(false)->where(array('id'=>$ruleData['id']))->update($newRule);
$ruleId=$ruleData['id']; $ruleId=$ruleData['id'];
} }
$this->dispatchJump(true,$ruleId); $this->dispatchJump(true,$ruleId);

109
SkycaijiApp/admin/controller/Task.php
View File

@ -12,6 +12,7 @@
namespace skycaiji\admin\controller; namespace skycaiji\admin\controller;
use think\Loader; use think\Loader;
use skycaiji\admin\model\CacheModel;
class Task extends BaseController { class Task extends BaseController {
public function indexAction(){ public function indexAction(){
@ -84,11 +85,11 @@ class Task extends BaseController {
$cond['tg_id']=0; $cond['tg_id']=0;
} }
$taskList=$mtask->where($cond)->order($orderBy)->paginate($limit,false,array('query'=>$search)); $taskList=$mtask->where($cond)->order($orderBy)->paginate($limit,false,paginate_auto_config());
$pagenav=$taskList->render(); $pagenav=$taskList->render();
$taskList=$taskList->all(); $taskList=$taskList->all();
}else{ }else{
$taskList=$mtask->where($cond)->order($orderBy)->paginate($limit,false,array('query'=>$search)); $taskList=$mtask->where($cond)->order($orderBy)->paginate($limit,false,paginate_auto_config());
$pagenav=$taskList->render(); $pagenav=$taskList->render();
$taskList=$taskList->all(); $taskList=$taskList->all();
if(!empty($taskList)){ if(!empty($taskList)){
@ -113,7 +114,7 @@ class Task extends BaseController {
$count=$mtask->count(); $count=$mtask->count();
$limit=20; $limit=20;
$taskList=$mtask->order('sort desc')->paginate($limit,false,array('query'=>array('show'=>$show))); $taskList=$mtask->order('sort desc')->paginate($limit,false,paginate_auto_config());
$pagenav=$taskList->render(); $pagenav=$taskList->render();
$taskList=$taskList->all(); $taskList=$taskList->all();
$this->assign('taskList',$taskList); $this->assign('taskList',$taskList);
@ -128,7 +129,7 @@ class Task extends BaseController {
public function openListAction(){ public function openListAction(){
$tgid=input('tg_id/d',0); $tgid=input('tg_id/d',0);
$mtaskgroup=model('taskgroup'); $mtaskgroup=model('Taskgroup');
$mtask=model('Task'); $mtask=model('Task');
$subTgList=$mtaskgroup->where(array('parent_id'=>$tgid))->order('sort desc')->column('*'); $subTgList=$mtaskgroup->where(array('parent_id'=>$tgid))->order('sort desc')->column('*');
@ -183,7 +184,7 @@ class Task extends BaseController {
} }
} }
$mtask->allowField(true)->save($newData); $mtask->isUpdate(false)->allowField(true)->save($newData);
$tid=$mtask->id; $tid=$mtask->id;
if($tid>0){ if($tid>0){
$taskData=$mtask->getById($tid); $taskData=$mtask->getById($tid);
@ -203,7 +204,7 @@ class Task extends BaseController {
$importRele['task_id']=$taskData['id']; $importRele['task_id']=$taskData['id'];
$importRele['addtime']=NOW_TIME; $importRele['addtime']=NOW_TIME;
unset($importRele['id']); unset($importRele['id']);
model('Release')->allowField(true)->save($importRele); model('Release')->isUpdate(false)->allowField(true)->save($importRele);
} }
} }
@ -267,7 +268,7 @@ class Task extends BaseController {
} }
unset($newData['id']); unset($newData['id']);
if($mtask->allowField(true)->save($newData,array('id'=>intval($taskData['id'])))>=0){ if($mtask->strict(false)->where(array('id'=>intval($taskData['id'])))->update($newData)>=0){
$taskData=$mtask->getById($taskData['id']); $taskData=$mtask->getById($taskData['id']);
/*导入规则*/ /*导入规则*/
$ruleId=input('rule_id'); $ruleId=input('rule_id');
@ -402,41 +403,50 @@ class Task extends BaseController {
$this->success(lang('delete_success')); $this->success(lang('delete_success'));
}elseif($op=='auto'){ }elseif($op=='auto'){
$auto = min(1,input('auto/d',0)); $auto = min(1,input('auto/d',0));
$mtask->save(array('auto'=>$auto),array('id'=>$taskData['id'])); $mtask->strict(false)->where(array('id'=>$taskData['id']))->update(array('auto'=>$auto));
$this->success(lang('op_success')); $this->success(lang('op_success'));
}elseif($op=='saveall'){ }elseif($op=='saveall'){
$newsort=input('newsort/a'); $newsort=input('newsort/a');
if(is_array($newsort)&&count($newsort)>0){ if(is_array($newsort)&&count($newsort)>0){
foreach ($newsort as $key=>$val){ foreach ($newsort as $key=>$val){
$mtask->save(array('sort'=>intval($val)),array('id'=>intval($key))); $mtask->strict(false)->where('id',intval($key))->update(array('sort'=>intval($val)));
} }
} }
$this->success(lang('op_success'),'Task/list?show='.input('show')); $this->success(lang('op_success'),'Task/list?show='.input('show'));
} }
} }
/*删除后台任务*/
public function bkdeleteAction(){
$taskId=input('id/d',0);
$mcache=CacheModel::getInstance('backstage_task');
$mcache->db()->where('cname',$taskId)->delete();
$this->success();
}
/*执行任务采集*/ /*执行任务采集*/
public function collectAction(){ public function collectAction(){
$taskId=input('id/d',0);
if(input('?backstage')){ if(input('?backstage')){
if(!IS_CLI){
ignore_user_abort(true);
if($GLOBALS['config']['caiji']['server']=='cli'){
cli_command_exec('collect task --task_id '.$taskId);
exit();
}
}
ignore_user_abort(true); ignore_user_abort(true);
define('CLOSE_ECHO_MSG',true); define('CLOSE_ECHO_MSG',true);
$this->_backstage_task($taskId);
}else{ }else{
ignore_user_abort(false); ignore_user_abort(false);
} }
$taskId=input('id/d',0);
$this->_collect($taskId); $this->_collect($taskId);
} }
/*批量执行任务采集*/ /*批量执行任务采集*/
public function collectBatchAction(){ public function collectBatchAction(){
if(input('?backstage')){
ignore_user_abort(true);
define('CLOSE_ECHO_MSG',true);
}else{
ignore_user_abort(false);
}
$taskIds=input('ids'); $taskIds=input('ids');
if(empty($taskIds)){ if(empty($taskIds)){
$this->echo_msg('没有选中任务'); $this->echo_msg('没有选中任务');
@ -445,6 +455,23 @@ class Task extends BaseController {
$taskIds=explode(',', $taskIds); $taskIds=explode(',', $taskIds);
$taskIds=array_map('intval', $taskIds); $taskIds=array_map('intval', $taskIds);
if(input('?backstage')){
if(!IS_CLI){
ignore_user_abort(true);
if($GLOBALS['config']['caiji']['server']=='cli'){
cli_command_exec('collect batch --task_ids '.implode(',',$taskIds));
exit();
}
}
ignore_user_abort(true);
define('CLOSE_ECHO_MSG',true);
}else{
ignore_user_abort(false);
}
if($GLOBALS['config']['caiji']['timeout']>0){ if($GLOBALS['config']['caiji']['timeout']>0){
set_time_limit(60*$GLOBALS['config']['caiji']['timeout']); set_time_limit(60*$GLOBALS['config']['caiji']['timeout']);
}else{ }else{
@ -469,7 +496,38 @@ class Task extends BaseController {
$this->echo_msg('所有任务采集完毕！','green'); $this->echo_msg('所有任务采集完毕！','green');
} }
/*将任务标记为后台运行*/
public function _backstage_task($taskId){
$GLOBALS['backstage_task_runtime']=time();
if(model('Task')->where('id',$taskId)->count()>0){
$mcache=CacheModel::getInstance('backstage_task');
$mcache->db()->strict(false)->insert(array(
'cname'=>$taskId,
'dateline'=>$GLOBALS['backstage_task_runtime'],
'ctype'=>0,
'data'=>''
),true);
if(!isset($GLOBALS['backstage_task_ids'])){
$GLOBALS['backstage_task_ids']=array();
}
$GLOBALS['backstage_task_ids'][$taskId]=$taskId;
static $registered=false;
if(!$registered){
register_shutdown_function(function(){
if(!empty($GLOBALS['backstage_task_ids'])&&is_array($GLOBALS['backstage_task_ids'])){
$mcache=\skycaiji\admin\model\CacheModel::getInstance('backstage_task');
$mcache->db()->strict(false)->where('cname','in',$GLOBALS['backstage_task_ids'])->update(array('ctype'=>1,'data'=>time()));
}
});
$registered=true;
}
}
}
/*单个任务采集*/ /*单个任务采集*/
public function _collect($taskId){ public function _collect($taskId){
static $setted_timeout=null; static $setted_timeout=null;
@ -516,7 +574,7 @@ class Task extends BaseController {
exit(); exit();
} }
$releData=$releData->toArray(); $releData=$releData->toArray();
$mtask->save(array('caijitime'=>NOW_TIME),array('id'=>$taskData['id'])); $mtask->strict(false)->where('id',$taskData['id'])->update(array('caijitime'=>NOW_TIME));
$acoll=controller('admin/C'.strtolower($collData['module']),'event'); $acoll=controller('admin/C'.strtolower($collData['module']),'event');
$acoll->init($collData); $acoll->init($collData);
$arele=controller('admin/R'.strtolower($releData['module']),'event'); $arele=controller('admin/R'.strtolower($releData['module']),'event');
@ -602,7 +660,7 @@ class Task extends BaseController {
$this->echo_msg('总共需采集'.$caijiNum.'条数据','black'); $this->echo_msg('总共需采集'.$caijiNum.'条数据','black');
} }
foreach ($taskList as $taskData){ foreach ($taskList as $taskData){
$mtask->where('id',$taskData['id'])->update(array('caijitime'=>time())); $mtask->strict(false)->where('id',$taskData['id'])->update(array('caijitime'=>time()));
$collData=$mcoll->where(array('task_id'=>$taskData['id'],'module'=>$taskData['module']))->find(); $collData=$mcoll->where(array('task_id'=>$taskData['id'],'module'=>$taskData['module']))->find();
$releData=$mrele->where(array('task_id'=>$taskData['id']))->find(); $releData=$mrele->where(array('task_id'=>$taskData['id']))->find();
if(empty($collData)||empty($releData)){ if(empty($collData)||empty($releData)){
@ -617,6 +675,11 @@ class Task extends BaseController {
$this->echo_msg('任务：'.$taskData['name'].'，发布方式为API接口，跳过执行','orange'); $this->echo_msg('任务：'.$taskData['name'].'，发布方式为API接口，跳过执行','orange');
continue; continue;
} }
if(input('?backstage')){
$this->_backstage_task($taskData['id']);
}
$taskData['config']=unserialize($taskData['config']); $taskData['config']=unserialize($taskData['config']);
$mtask->loadConfig($taskData['config']); $mtask->loadConfig($taskData['config']);
@ -661,7 +724,7 @@ class Task extends BaseController {
} }
}while($field_list!='completed'); }while($field_list!='completed');
} }
if(empty($all_field_list)){ if(empty($all_field_list)){
$this->echo_msg('任务：'.$taskData['name'].' 没有采集到数据','orange'); $this->echo_msg('任务：'.$taskData['name'].' 没有采集到数据','orange');
}else{ }else{

16
SkycaijiApp/admin/controller/Taskgroup.php
View File

@ -39,14 +39,14 @@ class Taskgroup extends BaseController {
$count=$mtaskgroup->where($cond)->count(); $count=$mtaskgroup->where($cond)->count();
if($count>0){ if($count>0){
$parentList=$mtaskgroup->where($cond)->order('sort desc')->paginate($limit,false,array('query'=>$search)); $parentList=$mtaskgroup->where($cond)->order('sort desc')->paginate($limit,false,paginate_auto_config());
} }
}else{ }else{
$cond=array('parent_id'=>0); $cond=array('parent_id'=>0);
$count=$mtaskgroup->where($cond)->count(); $count=$mtaskgroup->where($cond)->count();
if($count>0){ if($count>0){
$parentList=$mtaskgroup->where($cond)->order('sort desc')->paginate($limit,false,array('query'=>$search)); $parentList=$mtaskgroup->where($cond)->order('sort desc')->paginate($limit,false,paginate_auto_config());
$parentIds=array(); $parentIds=array();
foreach ($parentList->all() as $item){ foreach ($parentList->all() as $item){
$parentIds[$item['id']]=$item['id']; $parentIds[$item['id']]=$item['id'];
@ -91,7 +91,7 @@ class Taskgroup extends BaseController {
$this->error($validate->getError()); $this->error($validate->getError());
} }
$mtaskgroup->allowField(true)->save($newData); $mtaskgroup->isUpdate(false)->allowField(true)->save($newData);
$tgid=$mtaskgroup->id; $tgid=$mtaskgroup->id;
if($tgid>0){ if($tgid>0){
$this->success(lang('op_success'),input('referer','','trim')?input('referer','','trim'):('Taskgroup/edit?id='.$tgid)); $this->success(lang('op_success'),input('referer','','trim')?input('referer','','trim'):('Taskgroup/edit?id='.$tgid));
@ -148,7 +148,8 @@ class Taskgroup extends BaseController {
unset($newData['parent_id']); unset($newData['parent_id']);
} }
unset($newData['id']); unset($newData['id']);
$result=$mtaskgroup->allowField(true)->save($newData,array('id'=>intval($tgData['id'])));
$result=$mtaskgroup->strict(false)->where(array('id'=>intval($tgData['id'])))->update($newData);
if($result>=0){ if($result>=0){
$this->success(lang('op_success'),'Taskgroup/edit?id='.$tgData['id']); $this->success(lang('op_success'),'Taskgroup/edit?id='.$tgData['id']);
}else{ }else{
@ -199,8 +200,9 @@ class Taskgroup extends BaseController {
$this->error(lang('tg_exist_sub')); $this->error(lang('tg_exist_sub'));
}else{ }else{
$mtaskgroup->where(array('id'=>$id))->delete(); $mtaskgroup->where(array('id'=>$id))->delete();
$mtask->save(array('tg_id'=>0),array('tg_id'=>$id)); $mtask->strict(false)->where(array('tg_id'=>$id))->update(array('tg_id'=>0));
$this->success(lang('delete_success')); $this->success(lang('delete_success'));
} }
}elseif($op=='move'){ }elseif($op=='move'){
@ -219,7 +221,7 @@ class Taskgroup extends BaseController {
if($tgData['id']!=$parent_id){ if($tgData['id']!=$parent_id){
$mtaskgroup->save(array('parent_id'=>$parent_id),array('id'=>intval($tgData['id']))); $mtaskgroup->strict(false)->where(array('id'=>intval($tgData['id'])))->update(array('parent_id'=>$parent_id));
} }
$this->success(lang('op_success'),input('referer','','trim')); $this->success(lang('op_success'),input('referer','','trim'));
}else{ }else{
@ -243,7 +245,7 @@ class Taskgroup extends BaseController {
} }
if($deleteIds){ if($deleteIds){
$mtaskgroup->where(array('id'=>array('in',$deleteIds)))->delete(); $mtaskgroup->where(array('id'=>array('in',$deleteIds)))->delete();
$mtask->save(array('tg_id'=>0),array('tg_id'=>array('in',$deleteIds))); $mtask->strict(false)->where(array('tg_id'=>array('in',$deleteIds)))->update(array('tg_id'=>0));
} }
} }
$this->success(lang($hasSub?'tg_deleteall_has_sub':'op_success')); $this->success(lang($hasSub?'tg_deleteall_has_sub':'op_success'));

35
SkycaijiApp/admin/controller/Tool.php
View File

@ -66,16 +66,22 @@ class Tool extends BaseController {
public function logsAction(){ public function logsAction(){
$logPath=realpath(config('root_path').'/runtime/log'); $logPath=realpath(config('root_path').'/runtime/log');
$logList=array(); $logList=array();
$paths=scandir($logPath); if(!empty($logPath)){
foreach ($paths as $path){ $paths=scandir($logPath);
if($path!='.'&&$path!='..'){ if(!empty($paths)){
$pathFiles=scandir($logPath.'/'.$path); foreach ($paths as $path){
foreach ($pathFiles as $pathFile){ if($path!='.'&&$path!='..'){
if($pathFile!='.'&&$pathFile!='..'){ $pathFiles=scandir($logPath.'/'.$path);
$logList[$path][]=array( if(!empty($pathFiles)){
'name'=>$pathFile, foreach ($pathFiles as $pathFile){
'file'=>realpath($logPath.'/'.$path.'/'.$pathFile), if($pathFile!='.'&&$pathFile!='..'){
); $logList[$path][]=array(
'name'=>$pathFile,
'file'=>realpath($logPath.'/'.$path.'/'.$pathFile),
);
}
}
}
} }
} }
} }
@ -93,7 +99,7 @@ class Tool extends BaseController {
$this->error('不是日志文件'); $this->error('不是日志文件');
} }
$log=file_get_contents($file); $log=file_get_contents($file);
exit($log); exit('<pre>'.$log.'</pre>');
} }
/*文件校验*/ /*文件校验*/
public function checkfileAction(){ public function checkfileAction(){
@ -424,10 +430,11 @@ class Tool extends BaseController {
$json=get_html($url); $json=get_html($url);
} }
$this->success('','',array('json'=>$json)); $this->success('','',array('json'=>$json));
}else{
$GLOBALS['content_header']='JSON解析';
$GLOBALS['breadcrumb']=breadcrumb(array('JSON解析'));
return $this->fetch();
} }
$GLOBALS['content_header']='JSON解析';
$GLOBALS['breadcrumb']=breadcrumb(array('JSON解析'));
return $this->fetch();
} }
} }

6
SkycaijiApp/admin/controller/User.php
View File

@ -18,7 +18,7 @@ class User extends BaseController {
$page=max(1,$page); $page=max(1,$page);
$limit=20; $limit=20;
$count=$muser->count(); $count=$muser->count();
$userList=$muser->order('uid asc')->paginate($limit); $userList=$muser->order('uid asc')->paginate($limit,false,paginate_auto_config());
$pagenav = $userList->render(); $pagenav = $userList->render();
$this->assign('pagenav',$pagenav); $this->assign('pagenav',$pagenav);
@ -67,7 +67,7 @@ class User extends BaseController {
$this->error('您不能添加“'.$GLOBALS['user']['group']['name'].'”用户组'); $this->error('您不能添加“'.$GLOBALS['user']['group']['name'].'”用户组');
} }
$newData['regtime']=NOW_TIME; $newData['regtime']=NOW_TIME;
$muser->allowField(true)->save($newData); $muser->isUpdate(false)->allowField(true)->save($newData);
if($muser->uid>0){ if($muser->uid>0){
$this->success(lang('op_success'),'User/list'); $this->success(lang('op_success'),'User/list');
}else{ }else{
@ -140,7 +140,7 @@ class User extends BaseController {
unset($newData['groupid']); unset($newData['groupid']);
} }
$muser->allowField(true)->save($newData,array('uid'=>$uid)); $muser->strict(false)->where(array('uid'=>$uid))->update($newData);
$this->success(lang('op_success'),'User/list'); $this->success(lang('op_success'),'User/list');
}else{ }else{

139
SkycaijiApp/admin/event/Cpattern.php
View File

@ -951,7 +951,6 @@ class Cpattern extends Collector{
$base_url=$baseUrls[$urlMd5]; $base_url=$baseUrls[$urlMd5];
$domain_url=$domainUrls[$urlMd5]; $domain_url=$domainUrls[$urlMd5];
$val=''; $val='';
$field_func='field_module_'.$module; $field_func='field_module_'.$module;
if(method_exists($this, $field_func)){ if(method_exists($this, $field_func)){
@ -1013,6 +1012,7 @@ class Cpattern extends Collector{
$val=$this->$field_func($field_params,$html); $val=$this->$field_func($field_params,$html);
} }
} }
$vals=null; $vals=null;
if(is_array($val)){ if(is_array($val)){
@ -1148,17 +1148,6 @@ class Cpattern extends Collector{
} }
} }
public function relation_match_url($html,$config){
if(empty($config['reg_url_module'])){
$url=$this->match_rule($html,$config['reg_url'],$config['url_merge']);
}elseif('json'==$config['reg_url_module']){
$url=$this->rule_module_json_data(array('json'=>$config['reg_url'],'json_arr'=>'jsonencode'),json_decode($html,true));
}elseif('xpath'==$config['reg_url_module']){
$url=$this->rule_module_xpath_data(array('xpath'=>$config['reg_url'],'xpath_attr'=>'href'),$html);
}
return $url;
}
public function match_rule($html,$rule,$merge,$multi=false,$multi_str=''){ public function match_rule($html,$rule,$merge,$multi=false,$multi_str=''){
$val=''; $val='';
@ -1209,35 +1198,59 @@ class Cpattern extends Collector{
return ''; return '';
} }
$page=$relation_url['page']; if(empty($relation_url['page'])){
$pass=false;
$depth_pages=array();
$depth=0;
while(!empty($page)){
if($page==$name){ if(!isset($this->relation_url_list[$cont_url][$name])){
$relationUrl=$this->rule_match_urls($relation_url, $html);
$pass=true; $relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
break; $this->relation_url_list[$cont_url][$name]=$relationUrl;
}else{
$relationUrl=$this->relation_url_list[$cont_url][$name];
} }
if(empty($this->config['new_relation_urls'][$page])){ }else{
$pass=true;
break;
}
$depth++;
$depth_pages[$depth]=$page;
$page=$this->config['new_relation_urls'][$page]['page'];
}
if($pass){
return ''; $page=$relation_url['page'];
} $pass=false;
$relationUrl=$this->relation_match_url($html,$relation_url); $depth_pages=array();
$this->relation_url_list[$cont_url][$relation_url['page']]=$relationUrl; $depth=0;
if(!empty($depth_pages)){ while(!empty($page)){
if($page==$name){
$pass=true;
break;
}
if(empty($this->config['new_relation_urls'][$page])){
$pass=true;
break;
}
$depth++;
$depth_pages[$depth]=$page;
$page=$this->config['new_relation_urls'][$page]['page'];
}
if($pass){
return '';
}
krsort($depth_pages); krsort($depth_pages);
$contPage=reset($depth_pages);
$relationUrl='';
if(isset($contPage)){
if(!isset($this->relation_url_list[$cont_url][$contPage])){
$relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$contPage], $html);
$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
$this->relation_url_list[$cont_url][$contPage]=$relationUrl;
}else{
$relationUrl=$this->relation_url_list[$cont_url][$contPage];
}
}
$depth_pages=array_slice($depth_pages, 1);
$depth_pages=is_array($depth_pages)?$depth_pages:array();
$depth_pages[]=$relation_url['name'];
foreach ($depth_pages as $page){ foreach ($depth_pages as $page){
if(empty($relationUrl)){ if(empty($relationUrl)){
@ -1250,13 +1263,16 @@ class Cpattern extends Collector{
return ''; return '';
} }
$relationUrl=$this->relation_match_url($relationHtml,$this->config['new_relation_urls'][$page]); $relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$page],$relationHtml);
$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
$this->relation_url_list[$cont_url][$page]=$relationUrl; $this->relation_url_list[$cont_url][$page]=$relationUrl;
}else{ }else{
$relationUrl=$this->relation_url_list[$cont_url][$page]; $relationUrl=$this->relation_url_list[$cont_url][$page];
} }
} }
} }
@ -1431,9 +1447,11 @@ class Cpattern extends Collector{
$vals=''; $vals='';
if(!empty($field_params['xpath'])){ if(!empty($field_params['xpath'])){
$dom=new \DOMDocument; $dom=new \DOMDocument;
$libxml_previous_state = libxml_use_internal_errors(true);
@$dom->loadHTML('<meta http-equiv="Content-Type" content="text/html;charset=utf-8">'.$html); @$dom->loadHTML('<meta http-equiv="Content-Type" content="text/html;charset=utf-8">'.$html);
$dom->normalize(); $dom->normalize();
$xPath = new \DOMXPath($dom); $xPath = new \DOMXPath($dom);
$xpath_attr=strtolower($field_params['xpath_attr']); $xpath_attr=strtolower($field_params['xpath_attr']);
@ -1525,6 +1543,9 @@ class Cpattern extends Collector{
break; break;
} }
} }
libxml_clear_errors();
} }
return $vals; return $vals;
} }
@ -2123,6 +2144,18 @@ class Cpattern extends Collector{
} }
} }
} }
if(input('?backstage')){
$backstageDate=CacheModel::getInstance('backstage_task')->db()->field('dateline')->where('cname',$this->collector['task_id'])->find();
if(empty($backstageDate)||$GLOBALS['backstage_task_runtime']<$backstageDate['dateline']){
unset($GLOBALS['backstage_task_ids'][$this->collector['task_id']]);
exit('终止进程');
}
}
if($mcacheCont->getCount($md5_cont_url)>0){ if($mcacheCont->getCount($md5_cont_url)>0){
$this->used_cont_urls[$md5_cont_url]=1; $this->used_cont_urls[$md5_cont_url]=1;
@ -2257,7 +2290,9 @@ class Cpattern extends Collector{
/*采集,return false表示终止采集*/ /*采集,return false表示终止采集*/
public function collect($num=10){ public function collect($num=10){
define('IS_COLLECTING', 1); if(!defined('IS_COLLECTING')){
define('IS_COLLECTING', 1);
}
@session_start(); @session_start();
\think\Session::pause(); \think\Session::pause();
@ -2489,6 +2524,34 @@ class Cpattern extends Collector{
} }
} }
return $urls; return $urls;
}if(preg_match('/\{json\:([^\}]*)\}/i',$url,$match)){
$url=preg_replace('/\{json\:([^\}]*)\}/i','',$url);
$jsonRule=trim($match[1]);
if(is_null($jsonRule)||$jsonRule==''){
$jsonRule='*';
}
$jsonData=$this->get_html($url);
$jsonData=json_decode($jsonData,true);
if(!empty($jsonData)&&is_array($jsonData)){
$urls=$this->rule_module_json_data(array('json'=>$jsonRule,'json_arr'=>'_original_'),$jsonData);
if(!is_array($urls)){
$urls=array($urls);
}
foreach ($urls as $k=>$v){
if(!is_string($v)||!preg_match('/^\w+\:\/\//i', $v)){
unset($urls[$k]);
}
}
if(!empty($urls)&&is_array($urls)){
$urls=array_unique($urls);
$urls=array_values($urls);
}
return $urls;
}
}elseif(preg_match('/[\r\n]/', $url)){ }elseif(preg_match('/[\r\n]/', $url)){
if(preg_match_all('/^\w+\:\/\/[^\r\n]+/im',$url,$urls)){ if(preg_match_all('/^\w+\:\/\/[^\r\n]+/im',$url,$urls)){

12
SkycaijiApp/admin/event/Rcms.php
View File

@ -12,6 +12,7 @@
/*发布设置:本地cms*/ /*发布设置:本地cms*/
namespace skycaiji\admin\event; namespace skycaiji\admin\event;
class Rcms extends Release{ class Rcms extends Release{
protected $rele_cms_list=array();
/** /**
* 设置页面post过来的config * 设置页面post过来的config
* @param unknown $config * @param unknown $config
@ -60,8 +61,15 @@ class Rcms extends Release{
} }
} }
$releCms=model('ReleaseApp')->appImportClass($this->config['cms']['app'],'cms');
$releCms->init(null,$this->release); $releCms=md5($this->config['cms']['app'].'__cms__'.serialize($this->release));
if(!isset($this->rele_cms_list[$releCms])){
$this->rele_cms_list[$releCms]=model('ReleaseApp')->appImportClass($this->config['cms']['app'],'cms');
$this->rele_cms_list[$releCms]->init(null,$this->release);
}
$releCms=$this->rele_cms_list[$releCms];
$addedNum=0; $addedNum=0;
foreach ($collFieldsList as $collFieldsKey=>$collFields){ foreach ($collFieldsList as $collFieldsKey=>$collFields){

4
SkycaijiApp/admin/event/Rdb.php
View File

@ -51,6 +51,7 @@ class Rdb extends Release{
$db_config=$this->get_db_config($this->config['db']); $db_config=$this->get_db_config($this->config['db']);
$db_key=md5(serialize($db_config)); $db_key=md5(serialize($db_config));
if(empty($this->db_conn_list[$db_key])){ if(empty($this->db_conn_list[$db_key])){
$mdb=new DbCommon($db_config); $mdb=new DbCommon($db_config);
$mdb=$mdb->db(); $mdb=$mdb->db();
$this->db_conn_list[$db_key]=$mdb; $this->db_conn_list[$db_key]=$mdb;
@ -207,7 +208,8 @@ class Rdb extends Release{
'db_host' => $config_db['host'], 'db_host' => $config_db['host'],
'db_port' => $config_db['port'], 'db_port' => $config_db['port'],
'db_charset' => $config_db['charset'], 'db_charset' => $config_db['charset'],
'db_name' => $config_db['name'] 'db_name' => $config_db['name'],
); );
if(strcasecmp($db_config['db_charset'], 'utf-8')===0){ if(strcasecmp($db_config['db_charset'], 'utf-8')===0){

11
SkycaijiApp/admin/event/Rdiy.php
View File

@ -12,6 +12,7 @@
/*发布设置:diy*/ /*发布设置:diy*/
namespace skycaiji\admin\event; namespace skycaiji\admin\event;
class Rdiy extends Release{ class Rdiy extends Release{
protected $rele_diy_list=array();
/** /**
* 设置页面post过来的config * 设置页面post过来的config
* @param unknown $config * @param unknown $config
@ -54,8 +55,14 @@ class Rdiy extends Release{
if(!empty($appName)){ if(!empty($appName)){
if(model('ReleaseApp')->appFileExists($appName,'diy')){ if(model('ReleaseApp')->appFileExists($appName,'diy')){
$releDiy=model('ReleaseApp')->appImportClass($appName,'diy');
$releDiy->init($this->release); $releDiy=md5($appName.'__diy__'.serialize($this->release));
if(!isset($this->rele_diy_list[$releDiy])){
$this->rele_diy_list[$releDiy]=model('ReleaseApp')->appImportClass($appName,'diy');
$this->rele_diy_list[$releDiy]->init($this->release);
}
$releDiy=$this->rele_diy_list[$releDiy];
}elseif(model('ReleaseApp')->oldFileExists($appName,'diy')){ }elseif(model('ReleaseApp')->oldFileExists($appName,'diy')){
$this->echo_msg(lang('release_upgrade')); $this->echo_msg(lang('release_upgrade'));

5
SkycaijiApp/admin/model/Collector.php
View File

@ -16,14 +16,15 @@ class Collector extends BaseModel{
public function add_new($data){ public function add_new($data){
$data['addtime']=NOW_TIME; $data['addtime']=NOW_TIME;
$data['uptime']=NOW_TIME; $data['uptime']=NOW_TIME;
$this->allowField(true)->save($data); $this->isUpdate(false)->allowField(true)->save($data);
return $this->id; return $this->id;
} }
public function edit_by_id($id,$data){ public function edit_by_id($id,$data){
unset($data['addtime']); unset($data['addtime']);
$data['uptime']=NOW_TIME; $data['uptime']=NOW_TIME;
$this->allowField(true)->save($data,array('id'=>$id));
$this->strict(false)->where(array('id'=>$id))->update($data);
} }
} }

10
SkycaijiApp/admin/model/DbCommon.php
View File

@ -30,7 +30,17 @@ class DbCommon{
'dsn'=>$config['db_dsn'], 'dsn'=>$config['db_dsn'],
'resultset_type'=>'array', 'resultset_type'=>'array',
'break_reconnect'=>true, 'break_reconnect'=>true,
'params'=>array(),
); );
if(!empty($GLOBALS['config']['site']['dblong'])){
$this->config['params'][\PDO::ATTR_PERSISTENT]=true;
}
if(isset($config['fields_strict'])){
$this->config['fields_strict']=$config['fields_strict'];
}
if($this->config['type']=='mysqli'){ if($this->config['type']=='mysqli'){

10
SkycaijiApp/admin/model/Proxyip.php
View File

@ -64,13 +64,13 @@ class Proxyip extends BaseModel {
if(!empty($this->setting['use'])){ if(!empty($this->setting['use'])){
if($this->setting['use']=='num'){ if($this->setting['use']=='num'){
$this->where('1=1')->update(array('num'=>0)); $this->strict(false)->where('1=1')->update(array('num'=>0));
}elseif($this->setting['use']=='time'){ }elseif($this->setting['use']=='time'){
$this->where('1=1')->update(array('time'=>0)); $this->strict(false)->where('1=1')->update(array('time'=>0));
} }
}else{ }else{
$this->where('1=1')->update(array('num'=>0)); $this->strict(false)->where('1=1')->update(array('num'=>0));
} }
$proxyipData=$this->where($cond)->find(); $proxyipData=$this->where($cond)->find();
} }
@ -94,7 +94,7 @@ class Proxyip extends BaseModel {
$upData['num']=$proxyipData['num']+1; $upData['num']=$proxyipData['num']+1;
} }
$this->allowField(true)->save($upData,array('ip'=>$proxyipData['ip'])); $this->strict(false)->where(array('ip'=>$proxyipData['ip']))->update($upData);
} }
return $proxyipData; return $proxyipData;
} }
@ -115,7 +115,7 @@ class Proxyip extends BaseModel {
$upData['invalid']=1; $upData['invalid']=1;
} }
$this->allowField(true)->save($upData,array('ip'=>$proxy_ip['ip'])); $this->strict(false)->where(array('ip'=>$proxy_ip['ip']))->update($upData);
} }
} }

4
SkycaijiApp/admin/model/ReleaseApp.php
View File

@ -27,12 +27,12 @@ class ReleaseApp extends BaseModel{
if(!empty($cmsData)){ if(!empty($cmsData)){
$this->allowField(true)->save($cms,array('module'=>'cms','app'=>$cms['app'])); $this->strict(false)->where(array('module'=>'cms','app'=>$cms['app']))->update($cms);
$success=true; $success=true;
}else{ }else{
$cms['addtime']=NOW_TIME; $cms['addtime']=NOW_TIME;
$this->save($cms); $this->isUpdate(false)->allowField(true)->save($cms);
$cms['id']=$this->id; $cms['id']=$this->id;
$success=$cms['id']>0?true:false; $success=$cms['id']>0?true:false;
} }

43
SkycaijiApp/admin/view/backstage/bk_task.html Normal file
View File

@ -0,0 +1,43 @@
<div id="win_backstage_task" class="nav-tabs-custom">
<ul class="nav nav-tabs">
<li class="active"><a href="#win_bk_task0" data-toggle="tab">运行中（<span>{$count0}</span>）</a></li>
<li><a href="#win_bk_task1" data-toggle="tab">已完成（<span>{$count1}</span>）</a></li>
</ul>
<div class="tab-content">
<div class="tab-pane active" id="win_bk_task0" data-op="task0">
</div>
<div class="tab-pane" id="win_bk_task1" data-op="task1">
</div>
</div>
</div>
<script type="text/javascript">
'use strict';//严格模式
//任务列表
function win_bk_task_list(op,url){
$('#win_bk_'+op).html('<img src="'+window.site_config.pub+'/static/images/load1.gif" />');
$.ajax({
type: 'get',
dataType: 'json',
async:true,
url:(url?url:ulink('admin/backstage/backstageTask?op='+op)),
success: function (data){
if(data.code==1&&data.data){
var count=parseInt(data.data.count);
$('#win_backstage_task').find('a[href="#win_bk_'+op+'"]').tab('show').find('span').text(count);//显示数量
$('#win_bk_'+op).html(data.data.html);
}else{
$('#win_bk_'+op).html('无任务');
}
backstageTask();//重置数量
}
});
}
//加载任务列表
$('#win_backstage_task a[href="#win_bk_task0"],#win_backstage_task a[href="#win_bk_task1"]').bind('click',function(){
var op=$($(this).attr('href')).attr('data-op');
win_bk_task_list(op);
});
$('#win_backstage_task a[href="#win_bk_task0"]').trigger('click');//默认触发
</script>

56
SkycaijiApp/admin/view/backstage/bk_task_collected.html Normal file
View File

@ -0,0 +1,56 @@
{if $list}
<div class="table-responsive">
<table class="table table-bordered table-hover datatable" style="margin-bottom:0;">
<thead>
<tr>
<th>#</th>
<th>已发布至</th>
<th>发布方式</th>
<th>添加时间</th>
<th class="url">源网址</th>
</tr>
</thead>
<tbody>
{php}$curIx=0;{/php}
{foreach name="list" item="item"}
<tr>
<td>{php}$curIx++;{/php}{$curIx}</td>
<td class="desc">
{if condition="!empty($item['error'])"}
失败：{$item['error']}
{else /}
{$item['target']} {$item['desc']?$item['desc']:''}
{/if}
</td>
<td>{:lang('collected_rele_'.$item['release'])}</td>
<td>{:date('Y-m-d H:i:s',$item['addtime'])}</td>
<td class="url"><a href="{$item['url']}" target="_blank">{$item['url']}</a></td>
</tr>
{/foreach}
</tbody>
</table>
</div>
{if $pagenav}
<div class="box-body">{$pagenav}</div>
{/if}
{else /}
<div class="box-body">无数据</div>
{/if}
<script type="text/javascript">
'use strict';//严格模式
$('[id^="win_bk_t_list_"] .pagination').addClass('pagination-sm');
$('[id^="win_bk_t_list_"] .pagination a').bind('click',function(){
var taskId=$(this).parents('[id^="win_bk_t_list_"]').eq(0).attr('data-task-id');
if(window.interval_bk_tasks[taskId]){
//停止自动刷新
clearInterval(window.interval_bk_tasks[taskId]);
}
win_bk_task_collected(taskId,$(this).attr('href'));
return false;
});
{if $taskStatus}
//正在采集任务已完成
$('#win_bk_task_list_0').find('a[href="#win_bk_t_list_{$taskId}"]').find('small').text('已结束');
{/if}
</script>

99
SkycaijiApp/admin/view/backstage/bk_task_list.html Normal file
View File

@ -0,0 +1,99 @@
<div id="win_bk_task_list_{$taskType}">
{if $list}
{foreach $list as $task}
<div class="panel box box-default" style="margin-bottom:0;">
<div class="box-header with-border">
<h4 class="box-title">
<a data-toggle="collapse" data-parent="#win_bk_task_list_{$taskType}" href="#win_bk_t_list_{$task['id']}" class="collapsed">
{$task['name']}
{if $taskType}
({$cacheList[$task['id']]['collected_count']})
<small>{$cacheList[$task['id']]['enddate']} 结束 </small>
{else /}
<small style="color:inherit;"><span class="fa fa-refresh fa-spin"></span></small>
{/if}
</a>
</h4>
<a href="javascript:;" class="fa fa-remove pull-right" data-task-id="{$task['id']}" style="font-size:90%;color:#ccc" title="删除"></a>
</div>
<div id="win_bk_t_list_{$task['id']}" data-task-id="{$task['id']}" class="panel-collapse collapse">
<div class="box-body">
</div>
</div>
</div>
{/foreach}
{if $pagenav}
<div style="margin-top:15px;">{$pagenav}</div>
{/if}
{else /}
无任务
{/if}
</div>
<script type="text/javascript">
'use strict';//严格模式
//任务已采集数据
function win_bk_task_collected(taskId,url){
$('#win_bk_t_list_'+taskId+' .box-body').html('<img src="'+window.site_config.pub+'/static/images/load1.gif" />');
$.ajax({
type: 'get',
dataType: 'html',
async:true,
url:(url?url:ulink('admin/backstage/backstageTask?op=collected&tid='+taskId)),
success: function (data){
$('#win_bk_t_list_'+taskId).html(data);
}
});
}
if(!window.interval_bk_tasks){
//定时器全局变量
window.interval_bk_tasks={};
}
$('#win_bk_task_list_0 a[data-parent="#win_bk_task_list_0"]').bind('click',function(){
var taskId=$($(this).attr('href')).attr('data-task-id');
win_bk_task_collected(taskId);//先读取一次
//正在采集自动刷新
if(window.interval_bk_tasks[taskId]){
//先停止
clearInterval(window.interval_bk_tasks[taskId]);
}
window.interval_bk_tasks[taskId]=setInterval(function(){
if($('#win_bk_t_list_'+taskId).is(":hidden")){
//元素不可见停止刷新
clearInterval(window.interval_bk_tasks[taskId]);
}else{
win_bk_task_collected(taskId);
}
},3000);
});
$('#win_bk_task_list_1 a[data-parent="#win_bk_task_list_1"]').bind('click',function(){
//已采集任务
var taskId=$($(this).attr('href')).attr('data-task-id');
win_bk_task_collected(taskId);
});
$('[id^="win_bk_task_list_"] .fa-remove').bind('click',function(){
//删除后台任务
var obj=$(this);
var taskId=$(this).attr('data-task-id');
$.ajax({
type: 'get',
dataType: 'json',
async:true,
url:ulink('admin/task/bkdelete?id='+taskId),
success: function (data){
obj.parents('.panel').remove();
}
});
});
$('#win_bk_task_list_{$taskType} .pagination').addClass('pagination-sm');
$('#win_bk_task_list_{$taskType} .pagination a').bind('click',function(){
var op=$(this).parents('.tab-pane[data-op]').eq(0).attr('data-op');
win_bk_task_list(op,$(this).attr('href'));
return false;
});
</script>

5
SkycaijiApp/admin/view/collected/list.html
View File

@ -5,9 +5,10 @@
{block name="content"} {block name="content"}
<div class="box"> <div class="box">
<div class="box-body"> <div class="box-body">
<div class="alert alert-warning alert-dismissible"> <div class="alert alert-warning alert-dismissible" style="padding-top:7px;padding-bottom:7px;line-height:20px;">
<button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button> <button type="button" class="close" data-dismiss="alert" aria-hidden="true" style="top:1px;">×</button>
已采集网址可防止重复采集，如无必要，请勿删除！ 已采集网址可防止重复采集，如无必要，请勿删除！
<button type="button" id="btn_clear_error" class="btn btn-sm btn-warning" style="padding:2px 10px;">一键清理失败的数据</button>
</div> </div>
<form id="form_list" method="post" ajax-submit="true" action="{:url('Collected/op')}"> <form id="form_list" method="post" ajax-submit="true" action="{:url('Collected/op')}">
<div class="table-responsive"> <div class="table-responsive">

1
SkycaijiApp/admin/view/collector/list_task.html
View File

@ -20,6 +20,7 @@
var name=$(this).text(); var name=$(this).text();
import_rule('collector:'+id,name); import_rule('collector:'+id,name);
}); });
$('#window_colls .pagination').addClass('pagination-sm');
$('#window_colls .pagination a').bind('click',function(){ $('#window_colls .pagination a').bind('click',function(){
windowModal('导入规则',$(this).attr('href')); windowModal('导入规则',$(this).attr('href'));
return false; return false;

2
SkycaijiApp/admin/view/common/browser_is_old.html
View File

@ -1,4 +1,4 @@
{if condition="$GLOBALS['browser_is_old']"} {if condition="!empty($GLOBALS['browser_is_old'])"}
<div class="alert alert-error alert-dismissible fade in" role="alert" style="position:fixed;top:0;z-index:9999;width:100%;"> <div class="alert alert-error alert-dismissible fade in" role="alert" style="position:fixed;top:0;z-index:9999;width:100%;">
<button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">×</span></button> <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">×</span></button>
您的浏览器版本过低，不能完全展现页面效果，请使用谷歌、火狐或IE9及以上版本的浏览器！ 您的浏览器版本过低，不能完全展现页面效果，请使用谷歌、火狐或IE9及以上版本的浏览器！

6
SkycaijiApp/admin/view/common/main.html
View File

@ -17,6 +17,12 @@
</a> </a>
<div class="navbar-custom-menu"> <div class="navbar-custom-menu">
<ul class="nav navbar-nav"> <ul class="nav navbar-nav">
<li class="dropdown tasks-menu">
<a href="javascript:;" id="menu_backstage_task">
<i class="fa fa-flag-o"></i>
<span class="label label-warning"></span>
</a>
</li>
<li class="dropdown user user-menu"> <li class="dropdown user user-menu">
<a href="#" class="dropdown-toggle" data-toggle="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown">
<span class="glyphicon glyphicon-user"></span> <span class="glyphicon glyphicon-user"></span>

4
SkycaijiApp/admin/view/cpattern/set.html
View File

@ -561,8 +561,8 @@
<span class="caret"></span> <span class="caret"></span>
</button> </button>
<ul class="dropdown-menu" style="width:100%;text-align:center;" aria-labelledby="dropdownMenuTestUrl"> <ul class="dropdown-menu" style="width:100%;text-align:center;" aria-labelledby="dropdownMenuTestUrl">
<li><a href="{:url('Cpattern/test?op=cont_url&coll_id='.$collData['id'])}" target="_blank" onclick="windowModal('测试抓取',$(this).attr('href'),{lg:1});return false;">测试抓取数据</a></li> <li><a href="{:url('Cpattern/test?op=cont_url&coll_id='.$collData['id'])}" target="_blank">测试抓取数据</a></li>
<li><a href="{:url('Cpattern/test?op=match&coll_id='.$collData['id'])}" target="_blank" onclick="windowModal('模拟匹配',$(this).attr('href'),{lg:1});return false;">模拟匹配数据</a></li> <li><a href="{:url('Cpattern/test?op=match&coll_id='.$collData['id'])}" target="_blank">模拟匹配数据</a></li>
</ul> </ul>
</div> </div>
</div> </div>

9
SkycaijiApp/admin/view/cpattern/source.html
View File

@ -13,6 +13,7 @@
<li class="active"><a href="#tab_custom" data-toggle="tab" source-type="custom">手工指定</a></li> <li class="active"><a href="#tab_custom" data-toggle="tab" source-type="custom">手工指定</a></li>
<li class=""><a href="#tab_batch" data-toggle="tab" source-type="batch">批量生成</a></li> <li class=""><a href="#tab_batch" data-toggle="tab" source-type="batch">批量生成</a></li>
<li class=""><a href="#tab_large" data-toggle="tab" source-type="large">大量网址</a></li> <li class=""><a href="#tab_large" data-toggle="tab" source-type="large">大量网址</a></li>
<li class=""><a href="#tab_api" data-toggle="tab" source-type="api">API动态获取</a></li>
</ul> </ul>
<div class="tab-content"> <div class="tab-content">
@ -69,6 +70,14 @@
<p>大量网址，一行一条(http://或https://开头)</p> <p>大量网址，一行一条(http://或https://开头)</p>
<textarea name="source[large_urls]" class="form-control" rows="5"></textarea> <textarea name="source[large_urls]" class="form-control" rows="5"></textarea>
</div> </div>
<div class="tab-pane fade" id="tab_api" style="padding:10px 0;">
<p>API接口网址</p>
<input name="source[api]" class="form-control" />
<p class="help-block">通过api接口动态获取网址</p>
<p>网址获取规则</p>
<input name="source[api_json]" class="form-control" placeholder="默认获取json根节点的值" />
<p class="help-block">json规则： a.b.c 通配符 *</p>
</div>
</div> </div>
<div class="form-group form-group-sm"> <div class="form-group form-group-sm">

109
SkycaijiApp/admin/view/cpattern/test_cont_url_ajax.html
View File

@ -1,3 +1,13 @@
<style type="text/css">
#win_json_tree{width:100%;overflow:hidden;}
#win_json_tree ul{list-style:none;margin:0px;padding:0px;padding-left:20px;}
#win_json_tree li{padding:0;margin:0;}
#win_json_tree .tree{cursor:pointer;margin-left:-16px;margin-right:2px;}
#win_json_tree .node{color:#7F007F;}
#win_json_tree .val{}
</style>
<script src="__PUBLIC__/static/js/admin/json_tree.js?{$Think.config.html_v}"></script>
<div id="win_test_cont_url"> <div id="win_test_cont_url">
<div class="form-group"> <div class="form-group">
<div class="input-group"> <div class="input-group">
@ -18,10 +28,42 @@
<div class="input-group-btn"><button type="button" class="btn btn-default test-sub">确定</button></div> <div class="input-group-btn"><button type="button" class="btn btn-default test-sub">确定</button></div>
</div> </div>
</div> </div>
{if condition="!empty($input_urls)"}
<div id="win_get_fields_input">
<div class="panel panel-default">
<div class="panel-heading">有字段绑定了以下数据源，请输入网址</div>
<div class="panel-body">
{if condition="isset($input_urls['source_url'])"}
<div class="form-group">
<div class="input-group">
<div class="input-group-addon">起始页</div>
<input type="text" name="source_url" class="form-control" value="{$input_urls['source_url']}" />
</div>
</div>
{/if}
{if condition="is_array($input_urls['level_url'])"}
{foreach $input_urls['level_url'] as $level_url}
<div class="form-group">
<div class="input-group">
<div class="input-group-addon">多级页：{$level_url['name']}</div>
<input type="text" name="level_{$level_url['level']}" class="form-control" value="{$level_url['url']}" />
</div>
</div>
{/foreach}
{/if}
</div>
</div>
</div>
{/if}
<div class="vals"> <div class="vals">
</div> </div>
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
'use strict';//严格模式
//预览
function win_test_view(id){ function win_test_view(id){
$('#ifm_test_view').remove();//删除旧的 $('#ifm_test_view').remove();//删除旧的
var html=$('#'+id).val(); var html=$('#'+id).val();
@ -41,8 +83,8 @@ function win_test_view(id){
$("#ifm_test_view").contents().find('body').html(html); $("#ifm_test_view").contents().find('body').html(html);
} }
//分析元素
function test_elements(id){ function test_elements(id){
//分析元素
var cont_url=$('#win_test_cont_url input[name="url"]').val(); var cont_url=$('#win_test_cont_url input[name="url"]').val();
var url="{:url('Cpattern/test?op=_op_&coll_id='.$collData['id'].'&cont_url=_url_')}"; var url="{:url('Cpattern/test?op=_op_&coll_id='.$collData['id'].'&cont_url=_url_')}";
url=url.replace('_op_','elements').replace('_url_',encodeURIComponent(cont_url)); url=url.replace('_op_','elements').replace('_url_',encodeURIComponent(cont_url));
@ -53,6 +95,12 @@ $('#win_test_cont_url .dropdown-menu a[data-test]').bind('click',function(){
var btnObj=$(this).parents('.dropdown-menu').eq(0).siblings('button.dropdown-toggle').eq(0); var btnObj=$(this).parents('.dropdown-menu').eq(0).siblings('button.dropdown-toggle').eq(0);
btnObj.attr('data-test',$(this).attr('data-test')); btnObj.attr('data-test',$(this).attr('data-test'));
btnObj.find('em').eq(0).text($(this).text()); btnObj.find('em').eq(0).text($(this).text());
if($(this).attr('data-test')=='get_fields'){
//显示输入数据源网址
$('#win_get_fields_input').show();
}else{
$('#win_get_fields_input').hide();
}
}); });
{if condition="!empty($test)"} {if condition="!empty($test)"}
@ -72,7 +120,7 @@ $('#win_test_cont_url button.test-sub').bind('click',function(){
url=url.replace('_op_',testOp).replace('_url_',encodeURIComponent(cont_url)); url=url.replace('_op_',testOp).replace('_url_',encodeURIComponent(cont_url));
if(testOp=='elements'){ if(testOp=='elements'){
test_elements(); test_elements();
$('#win_test_cont_url .vals').html('页面效果所见即所得，点击元素可获取xpath值'); $('#win_test_cont_url .vals').html('已过滤所有脚本代码，页面所见即所得！');
}else if(testOp=='get_html'){ }else if(testOp=='get_html'){
//获取源码，防止源代码输出错误 //获取源码，防止源代码输出错误
$.ajax({ $.ajax({
@ -82,18 +130,36 @@ $('#win_test_cont_url button.test-sub').bind('click',function(){
success:function(data){ success:function(data){
if(htmlIsJson(data)){ if(htmlIsJson(data)){
//AJAX错误提示 //AJAX错误提示
data=eval('(' + data + ')'); var json=eval('(' + data + ')');
$('#win_test_cont_url .vals').html(data.msg); if(json.msg){
//json提示
$('#win_test_cont_url .vals').html(json.msg);
}else{
//json源码
var html='<label>源码：</label><textarea class="form-control" rows="5">'+data
+'</textarea><label style="margin-top:15px;">JSON解析：</label><div id="win_json_tree"></div>';
$('#win_test_cont_url .vals').html(html);
//加载json树形
jsonTree.treeId='#win_json_tree';
jsonTree.treeClass='.tree';
jsonTree.load(data);
}
}else{ }else{
var eleId=generateUUID(); var eleId=generateUUID();
data=data.replace(/\</g,'&lt;').replace(/\>/g,'&gt;');//编码 data=data.replace(/\</g,'&lt;').replace(/\>/g,'&gt;');//编码
html='<label>源码：</label><a href="javascript:;" onclick="test_elements()">分析元素</a><textarea id="' var html='<label>源码：</label><a href="javascript:;" onclick="test_elements()">分析元素</a><textarea id="'
+eleId+'" class="form-control" rows="20">'+data+'</textarea>'; +eleId+'" class="form-control" rows="20">'+data+'</textarea>';
$('#win_test_cont_url .vals').html(html); $('#win_test_cont_url .vals').html(html);
} }
} }
}); });
}else{ }else{
if(testOp=='get_fields'){
$('#win_get_fields_input').find('input').each(function(){
//附加数据源网址
url+='&'+$(this).attr('name')+'='+encodeURIComponent($(this).val());
});
}
$.ajax({ $.ajax({
type:'get', type:'get',
url:url, url:url,
@ -108,13 +174,33 @@ $('#win_test_cont_url button.test-sub').bind('click',function(){
} }
html+='</ul>'; html+='</ul>';
}else if(testOp=='get_fields'){ }else if(testOp=='get_fields'){
for(var i in data.data){ if(data.data.length>1){
if(data.data.length>1){ //循环数据
html+='<div class="panel panel-default"><div class="panel-heading"><h4 class="panel-title"><a data-toggle="collapse" aria-expanded="false">' html+='<div class="table-responsive"><table class="table table-bordered table-hover table-test-loop"><thead><tr><td></td>';
+'第'+(parseInt(i)+1)+'条数据</a></h4></div><div class="panel-collapse collapse in"><div class="panel-body">'; var fields=data.data[0];//第一条数据
for(var f in fields){
html+='<th>'+f+'</th>';
} }
html+='</tr></thead><tbody>';
var vals=data.data[i];//每条数据 var curNum=0;
for(var i in data.data){
curNum++;
var vals=data.data[i];//每条数据
html+='<tr><td>'+curNum+'</td>';
for(var f in vals){
if(vals[f]){
html+='<td><input type="text" value="'+(vals[f].replace(/"/g, '&quot;'))+'" /></td>';
}else{
html+='<td></td>';
}
}
html+='</tr>';
}
html+='</tbody>';
}else{
//一条数据
var vals=data.data[0];
for(var f in vals){ for(var f in vals){
var browse=''; var browse='';
var eleId=generateUUID(); var eleId=generateUUID();
@ -124,9 +210,6 @@ $('#win_test_cont_url button.test-sub').bind('click',function(){
} }
html+='<label>'+f+'：</label>'+browse+'<textarea id="'+eleId+'" class="form-control">'+vals[f]+'</textarea><br>'; html+='<label>'+f+'：</label>'+browse+'<textarea id="'+eleId+'" class="form-control">'+vals[f]+'</textarea><br>';
} }
if(data.data.length>1){
html+='</div></div></div>';
}
} }
}else if(testOp=='get_relation_urls'){ }else if(testOp=='get_relation_urls'){
html='<div class="page-header"><b>关联页网址</b></div><ul>'; html='<div class="page-header"><b>关联页网址</b></div><ul>';

32
SkycaijiApp/admin/view/cpattern/test_elements.html Normal file
View File

@ -0,0 +1,32 @@
{extend name="common:main" /}
{block name="cssjs"}
<style type="text/css">
#json_tree{width:100%;overflow:hidden;}
#json_tree ul{list-style:none;margin:0px;padding:0px;padding-left:20px;}
#json_tree li{padding:0;margin:0;}
#json_tree .tree{cursor:pointer;margin-left:-16px;margin-right:2px;}
#json_tree .node{color:#7F007F;}
#json_tree .val{}
</style>
<script src="__PUBLIC__/static/js/admin/json_tree.js?{$Think.config.html_v}"></script>
{/block}
{block name="content"}
<div class="box box-default">
<div class="box-body">
<div class="form-group">
<textarea id="txt_json" rows="3" class="form-control">{$html}</textarea>
</div>
<div id="json_tree"></div>
</div>
</div>
<script type="text/javascript">
'use strict';//严格模式
$(document).ready(function(){
jsonTree.treeId='#json_tree';
jsonTree.treeClass='.tree';
jsonTree.load($('#txt_json').val());
});
</script>
{/block}

92
SkycaijiApp/admin/view/cpattern/test_source_urls.html
View File

@ -1,19 +1,48 @@
<style type="text/css"> <style type="text/css">
#test_source_urls .source_url{padding:5px;} #test_source_urls .source_url{padding:5px;}
#test_source_urls p{margin:0;} #test_source_urls p{margin:0;}
#test_source_urls ol{padding-left:15px;}
</style> </style>
<div id="test_source_urls"> <div id="test_source_urls">
{if condition="empty($source_urls)"} {if condition="empty($source_urls)"}
没有起始页网址 没有起始页网址
{else /} {else /}
{foreach name="source_urls" item="source_url"} {if condition="$config['source_is_url']"}
<div class="source_url" url="{$source_url}"> <b>起始页已设置为内容页网址</b>
<p><b>抓取起始页面：</b>{$source_url}</p> {foreach $source_urls as $source=>$urls}
<p class="cont_urls_num"></p> <div class="source_url">
<div class="cont_urls_list"> <p>{$source}</p>
<ul style="padding-left:15px;white-space:normal;">
{if condition="empty($urls)"}
<li>无</li>
{elseif condition="is_array($urls)"}
{foreach $urls as $url}
<li>
[<a href="javascript:;" class="cont_ceshi" url="{$url}">测试</a>]
[<a href="javascript:;" class="cont_element" url="{$url}">元素</a>]
{$url}
</li>
{/foreach}
{else /}
<li>
[<a href="javascript:;" class="cont_ceshi" url="{$urls}">测试</a>]
[<a href="javascript:;" class="cont_element" url="{$urls}">元素</a>]
{$urls}
</li>
{/if}
</ul>
</div> </div>
</div> {/foreach}
{/foreach} {else /}
{foreach name="source_urls" item="source_url"}
<div class="source_url" url="{$source_url}">
<p><b>抓取起始页面：</b>{$source_url}</p>
<p class="cont_urls_num"></p>
<div class="cont_urls_list">
</div>
</div>
{/foreach}
{/if}
{/if} {/if}
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
@ -23,18 +52,51 @@ var url_ajax_requests=new Array();//ajax对象
$(document).ready(function(){ $(document).ready(function(){
$('#test_source_urls .source_url').each(function(index){ $('#test_source_urls .source_url').each(function(index){
//起始页开始测试 //起始页开始测试
var obj=$(this); {if condition="!$config['source_is_url']"}
var source_url=$(this).attr('url'); var source_url=$(this).attr('url');
{if condition="!empty($config['level_urls'])"} {if condition="!empty($config['level_urls'])"}
//有多级 //有多级
if(index<max_num){ if(index<max_num){
//测试前几页 //测试前几页
get_cont_urls(source_url,obj,1); get_cont_urls(source_url,$(this),1);
} }
{else /} {else /}
get_cont_urls(source_url,obj,0); get_cont_urls(source_url,$(this),0);
{/if} {/if}
{/if}
//测试
$('#test_source_urls').on('click','.cont_ceshi',function(){
var test_cont_url="{:url('Cpattern/test?op=cont_url&coll_id='.$collData['id'])}&cont_url=_url_";
test_cont_url=test_cont_url.replace('_url_',encodeURIComponent($(this).attr('url')));
var urls={};
$(this).parents('.source_url').each(function(){
var surl=$(this).attr('url');
if(surl){
if($(this).attr('level')){
//是级别
urls['level_'+$(this).attr('level')]=surl;
}else{
//是起始页
urls['source_url']=surl;
}
}
});
if(urls){
//加上多级和起始网址
for(var i in urls){
test_cont_url+='&'+i+'='+encodeURIComponent(urls[i]);
}
}
window.open(test_cont_url,'_test_cont');
});
//元素
$('#test_source_urls').on('click','.cont_element',function(){
var ele_cont_url="{:url('Cpattern/test?op=elements&coll_id='.$collData['id'])}&cont_url=_url_";
ele_cont_url=ele_cont_url.replace('_url_',encodeURIComponent($(this).attr('url')));
window.open(ele_cont_url,'_test_cont');
});
}); });
//展开收起 //展开收起
$('#test_source_urls').on('click','.cont_urls_num a',function(){ $('#test_source_urls').on('click','.cont_urls_num a',function(){
@ -71,7 +133,7 @@ $(document).ready(function(){
for(var i in urls){ for(var i in urls){
if(i<max_num){ if(i<max_num){
//测试前几页 //测试前几页
list+='<li class="source_url" url="'+urls[i]+'"><p><b>抓取第'+level+'级页面 “'+data.data.levelName+'”：</b>'+urls[i]+'</p><p class="cont_urls_num"></p><div class="cont_urls_list"></div></li>'; list+='<li class="source_url" url="'+urls[i]+'" level="'+level+'"><p><b>抓取第'+level+'级页面 “'+data.data.levelName+'”：</b>'+urls[i]+'</p><p class="cont_urls_num"></p><div class="cont_urls_list"></div></li>';
} }
} }
list+='</ul>'; list+='</ul>';
@ -86,11 +148,9 @@ $(document).ready(function(){
}else{ }else{
//无级别，显示测试网址 //无级别，显示测试网址
var test_cont_url="{:url('Cpattern/test?op=cont_url&coll_id='.$collData['id'])}&cont_url=_url_";
var ele_cont_url="{:url('Cpattern/test?op=elements&coll_id='.$collData['id'])}&cont_url=_url_";
for(var i in urls){ for(var i in urls){
list+='<li>[<a href="'+(test_cont_url.replace('_url_',encodeURIComponent(urls[i])))+'" target="_test_cont">测试</a>] ' list+='<li>[<a href="javascript:;" class="cont_ceshi" url="'+urls[i]+'">测试</a>] '
+'[<a href="'+(ele_cont_url.replace('_url_',encodeURIComponent(urls[i])))+'" target="_test_cont">元素</a>] '+urls[i]+'</li>'; +'[<a href="javascript:;" class="cont_element" url="'+urls[i]+'">元素</a>] '+urls[i]+'</li>';
} }
list+='</ul>'; list+='</ul>';

1
SkycaijiApp/admin/view/mystore/rules_task.html
View File

@ -12,6 +12,7 @@
var name=$(this).text(); var name=$(this).text();
import_rule('rule:'+id,name); import_rule('rule:'+id,name);
}); });
$('#window_rules .pagination').addClass('pagination-sm');
$('#window_rules .pagination a').bind('click',function(){ $('#window_rules .pagination a').bind('click',function(){
windowModal('导入规则',$(this).attr('href')); windowModal('导入规则',$(this).attr('href'));
return false; return false;

1
SkycaijiApp/admin/view/release/import.html
View File

@ -20,6 +20,7 @@
var name=$(this).text(); var name=$(this).text();
releaseClass.import(id,name); releaseClass.import(id,name);
}); });
$('#window_reles .pagination').addClass('pagination-sm');
$('#window_reles .pagination a').bind('click',function(){ $('#window_reles .pagination a').bind('click',function(){
windowModal('导入规则',$(this).attr('href')); windowModal('导入规则',$(this).attr('href'));
return false; return false;

27
SkycaijiApp/admin/view/setting/caiji.html
View File

@ -27,12 +27,21 @@
<option value="backstage">后台运行（默认）</option> <option value="backstage">后台运行（默认）</option>
<option value="visit">访问触发（推荐）</option> <option value="visit">访问触发（推荐）</option>
</select> </select>
<p class="help-block run-backstage">将在PHP后台开启一个持续进程用来运行采集，会占用一些服务器资源</p> <p class="help-block run-backstage">将在PHP后台开启一个持续进程用来运行采集，会占用一些服务器资源</p>
<p class="help-block run-visit" style="display:none;"> <p class="help-block run-visit" style="display:none;">
需在您的网站底部模板中加入一条html代码：<br/> 需在您的网站底部模板中加入一条html代码：<br/>
<em><b>&lt;script src="{:url('Admin/Index/caiji',null,false,true)}"&gt;&lt;/script&gt;</b></em> <em><b>&lt;script src="{:url('Admin/Index/caiji',null,false,true)}"&gt;&lt;/script&gt;</b></em>
<br/>添加后，用户访问页面时即可触发采集，该方法不会额外消耗服务器资源 <br/>添加后，用户访问页面时即可触发采集，该方法不会额外消耗服务器资源
</p> </p>
</div>
<div class="form-group" id="set_caiji_server" style="display:none;">
<label>采集模式</label>
<div class="input-group">
<label class="radio-inline"><input type="radio" name="server" value="">web服务器（默认）</label>
<label class="radio-inline"><input type="radio" name="server" value="cli">cli命令行（推荐）</label>
</div>
<p class="help-block server-">使用web服务器运行采集，会受web服务器运行超时影响导致采集中断，需修改web服务器的超时时间</p>
<p class="help-block server-cli" style="display:none;">使用php cli命令行模式运行采集</p>
</div> </div>
<div class="form-group"> <div class="form-group">
<label>{$Think.lang.set_caiji_num}</label> <label>{$Think.lang.set_caiji_num}</label>
@ -122,6 +131,7 @@
</form> </form>
<script type="text/javascript"> <script type="text/javascript">
'use strict';//严格模式
$('#form_set [name="auto"]').bind('click',function(){ $('#form_set [name="auto"]').bind('click',function(){
if($(this).val()>0){ if($(this).val()>0){
$('#set_caiji_run').show(); $('#set_caiji_run').show();
@ -135,6 +145,11 @@ $('#form_set [name="run"]').bind('change',function(){
$('#set_caiji_run .run-'+$(this).val()).show(); $('#set_caiji_run .run-'+$(this).val()).show();
}); });
$('#form_set [name="server"]').bind('click',function(){
$('#set_caiji_server .help-block').hide();
$('#set_caiji_server .server-'+$(this).val()).show();
});
$('#form_set [name="download_img"]').bind('click',function(){ $('#form_set [name="download_img"]').bind('click',function(){
if($(this).val()>0){ if($(this).val()>0){
$('#set_caiji_download_img').show(); $('#set_caiji_download_img').show();
@ -145,6 +160,8 @@ $('#form_set [name="download_img"]').bind('click',function(){
$('#form_set [name="auto"][value="{$caijiConfig["auto"]|intval}"]').trigger("click"); $('#form_set [name="auto"][value="{$caijiConfig["auto"]|intval}"]').trigger("click");
$('#form_set [name="run"]').val("{$caijiConfig['run']?$caijiConfig['run']:'backstage'}").trigger("change"); $('#form_set [name="run"]').val("{$caijiConfig['run']?$caijiConfig['run']:'backstage'}").trigger("change");
//$('#form_set [name="server"][value="{$caijiConfig["server"]}"]').prop('checked',true).trigger("click");
$('#form_set [name="real_time"][value="{$caijiConfig["real_time"]|intval}"]').prop('checked',true); $('#form_set [name="real_time"][value="{$caijiConfig["real_time"]|intval}"]').prop('checked',true);
$('#form_set [name="download_img"][value="{$caijiConfig["download_img"]|intval}"]').trigger("click"); $('#form_set [name="download_img"][value="{$caijiConfig["download_img"]|intval}"]').trigger("click");
$('#form_set [name="img_name"]').val('{$caijiConfig["img_name"]}'); $('#form_set [name="img_name"]').val('{$caijiConfig["img_name"]}');

37
SkycaijiApp/admin/view/setting/site.html
View File

@ -8,26 +8,26 @@
<form id="form_set" method="post" ajax-submit="true" role="form" action="{:url('Admin/Setting/site')}"> <form id="form_set" method="post" ajax-submit="true" role="form" action="{:url('Admin/Setting/site')}">
<div class="form-group"> <div class="form-group">
<label>关闭前台首页</label> <label>关闭前台首页</label>
<div class="radio"> <div class="input-group">
<label><input type="radio" name="hidehome" value="1">{$Think.lang.yes}</label> <label class="radio-inline"><input type="radio" name="hidehome" value="1">{$Think.lang.yes}</label>
<label><input type="radio" name="hidehome" value="0">{$Think.lang.no}</label> <label class="radio-inline"><input type="radio" name="hidehome" value="0">{$Think.lang.no}</label>
</div> </div>
<p class="help-block">不显示前台首页，访问首页直接跳转至后台</p> <p class="help-block">不显示前台首页，访问首页直接跳转至后台</p>
</div> </div>
<div class="form-group"> <div class="form-group">
<label>{$Think.lang.set_site_verifycode}</label> <label>{$Think.lang.set_site_verifycode}</label>
<div class="radio"> <div class="input-group">
<label><input type="radio" name="verifycode" value="1">{$Think.lang.yes}</label> <label class="radio-inline"><input type="radio" name="verifycode" value="1">{$Think.lang.yes}</label>
<label><input type="radio" name="verifycode" value="0">{$Think.lang.no}</label> <label class="radio-inline"><input type="radio" name="verifycode" value="0">{$Think.lang.no}</label>
</div> </div>
</div> </div>
<div class="form-group"> <div class="form-group">
<label>登录限制</label> <label>登录限制</label>
<div class="radio"> <div class="input-group">
<label><input type="radio" name="login[limit]" value="1">{$Think.lang.yes}</label> <label class="radio-inline"><input type="radio" name="login[limit]" value="1">{$Think.lang.yes}</label>
<label><input type="radio" name="login[limit]" value="0">{$Think.lang.no}</label> <label class="radio-inline"><input type="radio" name="login[limit]" value="0">{$Think.lang.no}</label>
</div> </div>
<div class="input-group" id="login_limit" style="display:none;"> <div class="input-group" id="login_limit" style="margin-top:10px;display:none;">
<span class="input-group-addon">登录失败</span> <span class="input-group-addon">登录失败</span>
<input type="number" name="login[failed]" class="form-control" value="{$siteConfig['login']['failed']}"> <input type="number" name="login[failed]" class="form-control" value="{$siteConfig['login']['failed']}">
<span class="input-group-addon">次，将被锁定</span> <span class="input-group-addon">次，将被锁定</span>
@ -35,6 +35,21 @@
<span class="input-group-addon">小时</span> <span class="input-group-addon">小时</span>
</div> </div>
</div>
<div class="form-group">
<label>数据库长连接</label>
<div class="input-group">
<label class="radio-inline"><input type="radio" name="dblong" value="1">{$Think.lang.yes}</label>
<label class="radio-inline"><input type="radio" name="dblong" value="0">{$Think.lang.no}</label>
</div>
<p class="help-block">使数据库持久连接</p>
</div>
<div class="form-group">
<label>关闭错误日志</label>
<div class="input-group">
<label class="radio-inline"><input type="radio" name="closelog" value="1">{$Think.lang.yes}</label>
<label class="radio-inline"><input type="radio" name="closelog" value="0">{$Think.lang.no}</label>
</div>
</div> </div>
<div class="form-group"> <div class="form-group">
<button type="submit" class="btn btn-primary">{$Think.lang.save}</button> <button type="submit" class="btn btn-primary">{$Think.lang.save}</button>
@ -53,6 +68,8 @@ $('#form_set [name="login[limit]"]').bind('click',function(){
$('#form_set [name="verifycode"][value="{$siteConfig['verifycode']|intval}"]').attr('checked', 'checked'); $('#form_set [name="verifycode"][value="{$siteConfig['verifycode']|intval}"]').attr('checked', 'checked');
$('#form_set [name="hidehome"][value="{$siteConfig['hidehome']|intval}"]').attr('checked', 'checked'); $('#form_set [name="hidehome"][value="{$siteConfig['hidehome']|intval}"]').attr('checked', 'checked');
$('#form_set [name="closelog"][value="{$siteConfig['closelog']|intval}"]').attr('checked', 'checked');
$('#form_set [name="dblong"][value="{$siteConfig['dblong']|intval}"]').attr('checked', 'checked');
$('#form_set [name="login[limit]"][value="{$siteConfig['login']['limit']|intval}"]').attr('checked', 'checked').trigger('click'); $('#form_set [name="login[limit]"][value="{$siteConfig['login']['limit']|intval}"]').attr('checked', 'checked').trigger('click');
</script> </script>

2
SkycaijiApp/admin/view/task/list_folder.html
View File

@ -43,6 +43,8 @@
全选 全选
</label> </label>
&nbsp;<a href="javascript:;" class="caiji-batch">批量采集</a> &nbsp;<a href="javascript:;" class="caiji-batch">批量采集</a>
<span class="sep">|</span>
<a href="javascript:;" class="houtai-batch">批量后台</a>
</td> </td>
</tr> </tr>
</tfoot> </tfoot>

1
SkycaijiApp/admin/view/task/list_import.html
View File

@ -14,6 +14,7 @@
var name=$(this).text(); var name=$(this).text();
import_task(id,name); import_task(id,name);
}); });
$('#window_tasks .pagination').addClass('pagination-sm');
$('#window_tasks .pagination a').bind('click',function(){ $('#window_tasks .pagination a').bind('click',function(){
windowModal('导入任务',$(this).attr('href')); windowModal('导入任务',$(this).attr('href'));
return false; return false;

8
SkycaijiApp/admin/view/task/list_list.html
View File

@ -38,12 +38,14 @@
{$item['caijitime']?date('Y-m-d H:i',$item['caijitime']):lang('none')} {$item['caijitime']?date('Y-m-d H:i',$item['caijitime']):lang('none')}
</label> </label>
&nbsp;<a href="javascript:;" class="caiji" item-id="{$item['id']}">{$Think.lang.caiji}</a> &nbsp;<a href="javascript:;" class="caiji" item-id="{$item['id']}">{$Think.lang.caiji}</a>
<span class="sep">|</span>
<a href="javascript:;" class="houtai" item-id="{$item['id']}">后台</a>
</td> </td>
<td> <td>
<a href="{:url('Collector/set?task_id='.$item['id'])}">规则</a> <a href="{:url('Collector/set?task_id='.$item['id'])}">规则</a>
&nbsp; <span class="sep">|</span>
<a href="{:url('Release/set?task_id='.$item['id'])}">发布</a> <a href="{:url('Release/set?task_id='.$item['id'])}">发布</a>
&nbsp; <span class="sep">|</span>
<a href="javascript:;" class="delete" item-id="{$item['id']}">{$Think.lang.delete}</a> <a href="javascript:;" class="delete" item-id="{$item['id']}">{$Think.lang.delete}</a>
</td> </td>
</tr> </tr>
@ -61,6 +63,8 @@
全选 全选
</label> </label>
&nbsp;<a href="javascript:;" class="caiji-batch">批量采集</a> &nbsp;<a href="javascript:;" class="caiji-batch">批量采集</a>
<span class="sep">|</span>
<a href="javascript:;" class="houtai-batch">批量后台</a>
</td> </td>
</tr> </tr>
{if condition="!empty($pagenav)"} {if condition="!empty($pagenav)"}

4
SkycaijiApp/admin/view/taskgroup/list.html
View File

@ -126,9 +126,9 @@ $(document).ready(function(){
confirmRight('{$Think.lang.confirm_delete}',function(){ confirmRight('{$Think.lang.confirm_delete}',function(){
$.ajax({ $.ajax({
type: "POST", type: "POST",
url: "{:url('Taskgroup/op?op=deleteall')}", url: "{:url('Taskgroup/op')}",
dataType: "json", dataType: "json",
data: $('#form_list').serialize(), data:$('#form_list').serialize()+'&op=deleteall',
success: function(data){ success: function(data){
data.code==1?toastr.success(data.msg):toastr.error(data.msg); data.code==1?toastr.success(data.msg):toastr.error(data.msg);
setTimeout("window.location.reload();",2500); setTimeout("window.location.reload();",2500);

76
SkycaijiApp/admin/view/tool/json_tree.html
View File

@ -8,8 +8,8 @@
#json_tree .node{color:#7F007F;} #json_tree .node{color:#7F007F;}
#json_tree .val{} #json_tree .val{}
</style> </style>
<script src="__PUBLIC__/static/js/admin/json_tree.js?{$Think.config.html_v}"></script>
{/block} {/block}
{block name="content"} {block name="content"}
<div class="box box-default"> <div class="box box-default">
@ -24,22 +24,17 @@
</div> </div>
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
'use strict';//严格模式
$(document).ready(function(){ $(document).ready(function(){
jsonTree.treeId='#json_tree';
jsonTree.treeClass='.tree';
$('#btn_tree').bind('click',function(){ $('#btn_tree').bind('click',function(){
var data=$('#txt_json').val(); var data=$('#txt_json').val();
if(htmlIsJson(data)){ if(htmlIsJson(data)){
//是json字符串 //是json字符串
data=JSON.parse(data); jsonTree.load(data);
if(data){
var html='<ul>';
for(var i in data){
html+=toolJsonTree(i,data[i]);
}
html+='</ul>';
$('#json_tree').html(html);
}
}else if((/^\w+\:\/\//).test(data)){ }else if((/^\w+\:\/\//).test(data)){
//是网址 //是网址
$.ajax({ $.ajax({
@ -49,16 +44,7 @@ $(document).ready(function(){
data: {'url':data}, data: {'url':data},
success: function (data) { success: function (data) {
data=data.data.json; data=data.data.json;
if(data){ jsonTree.load(data);
data=JSON.parse(data);
var html='<ul>';
for(var i in data){
html+=toolJsonTree(i,data[i]);
}
html+='</ul>';
$('#json_tree').html(html);
}
} }
}); });
}else{ }else{
@ -66,54 +52,6 @@ $(document).ready(function(){
} }
}); });
$('#json_tree').on('click','.tree',function(){
if($(this).hasClass('glyphicon-triangle-bottom')){
$(this).removeClass('glyphicon-triangle-bottom');
$(this).addClass('glyphicon-triangle-right');
var hasSub=false;
$(this).siblings('ul').children('li').each(function(){
var subTree=$(this).find('.tree').eq(0);
if(subTree.length>0){
subTree.removeClass('glyphicon-triangle-bottom').addClass('glyphicon-triangle-right').siblings('ul').hide();
hasSub=true;
}
});
if(!hasSub){
$(this).siblings('ul').hide();
}
}else{
$(this).removeClass('glyphicon-triangle-right');
$(this).addClass('glyphicon-triangle-bottom');
$(this).siblings('ul').show();
$(this).siblings('ul').children('li').each(function(){
$(this).find('.tree').eq(0).removeClass('glyphicon-triangle-right').addClass('glyphicon-triangle-bottom').siblings('ul').show();
});
}
});
}); });
function toolJsonTree(node,list){
var html='<li>';
var isList=false;//是列表
if(list){
if(typeof(list)=='object' && !$.isEmptyObject(list)){
isList=true;
}
}
if(isList){
html+='<span class="glyphicon glyphicon-triangle-bottom tree"></span><span class="node">'+node+'</span><ul>';
for(var i in list){
html+=toolJsonTree(i,list[i]);
}
html+='</ul>';
}else{
html+='<span class="node">'+node+'</span>: <span class="val">'+list+'</span>';
}
html+='</li>';
return html;
}
</script> </script>
{/block} {/block}

8
SkycaijiApp/command.php
View File

@ -8,7 +8,6 @@
| 使用协议 http://www.skycaiji.com/licenses | 使用协议 http://www.skycaiji.com/licenses
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
*/ */
@ -18,5 +17,8 @@
return [];
return [
'skycaiji\admin\command\Collect'
];

27
SkycaijiApp/common.php
View File

@ -10,7 +10,7 @@
*/ */
define('SKYCAIJI_VERSION', '2.0'); define('SKYCAIJI_VERSION', '2.1');
define('NOW_TIME', time()); define('NOW_TIME', time());
\think\Loader::addNamespace('plugin', realpath(ROOT_PATH.'plugin')); \think\Loader::addNamespace('plugin', realpath(ROOT_PATH.'plugin'));
\think\Loader::addNamespace('util',realpath(APP_PATH.'extend/util')); \think\Loader::addNamespace('util',realpath(APP_PATH.'extend/util'));
@ -383,4 +383,29 @@ function clear_dir($path,$passFiles=null){
function default_filter_func($str){ function default_filter_func($str){
return htmlspecialchars($str,ENT_QUOTES); return htmlspecialchars($str,ENT_QUOTES);
}
/*生成分页配置*/
function paginate_auto_config($path='',$queryParamsOrAuto=true){
if(empty($path)){
$path=request()->pathinfo();
}
if($queryParamsOrAuto==true){
$params=input('param.');
}else{
$params=is_array($queryParamsOrAuto)?$queryParamsOrAuto:array();
}
$params[config('paginate.var_page')]='-_-PAGE-_-';
$params=http_build_query($params);
$path.=(strpos($path,'?')!==false?'&':'?').$params;
$path=url($path);
$path=str_replace('-_-PAGE-_-', '[PAGE]', $path);
return array('path'=>$path);
} }

2
SkycaijiApp/common/behavior/Init.php
View File

@ -16,7 +16,7 @@ class Init{
public function run(){ public function run(){
session_start(); session_start();
if($_GET['m']&&$_GET['c']&&$_GET['a']){ if(isset($_GET['m'])&&isset($_GET['c'])&&isset($_GET['a'])){
$tourl=config('root_website'); $tourl=config('root_website');
if(stripos($tourl, '/index.php')===false){ if(stripos($tourl, '/index.php')===false){

12
SkycaijiApp/config.php
View File

@ -168,11 +168,11 @@ return [
'log' => [ 'log' => [
// 日志记录方式，内置 file socket 支持扩展 // 日志记录方式，内置 file socket 支持扩展
'type' => 'File', 'type' => '\util\Log',//使用自定义的日志驱动
// 日志保存目录 // 日志保存目录
'path' => LOG_PATH, 'path' => LOG_PATH,
// 日志记录级别 // 日志记录级别
'level' => [], 'level' => ['error'],
], ],
// +---------------------------------------------------------------------- // +----------------------------------------------------------------------
@ -243,12 +243,16 @@ return [
/**********************************自定义配置*********************************************/ /**********************************自定义配置*********************************************/
'html_v'=>'20190101',//css和js版本 'cli_cache_config'=>array('view_replace_str','root_path','app_path','plugin_path','root_url','root_website'),//cli模式下需要缓存的配置，否则会失效引起程序bug
'html_v'=>'20190301',//css和js版本
'root_path'=>realpath(ROOT_PATH),//根目录 'root_path'=>realpath(ROOT_PATH),//根目录
'app_path'=>realpath(APP_PATH),//APP目录 'app_path'=>realpath(APP_PATH),//APP目录
'plugin_path'=>realpath(ROOT_PATH.'plugin'),//插件目录 'plugin_path'=>realpath(ROOT_PATH.'plugin'),//插件目录
'root_url'=>rtrim(preg_replace('/\/index\.php.*/i','',Request::instance()->root()),'\/\\'),//网址根目录 'root_url'=>rtrim(preg_replace('/\/index\.php.*/i','',Request::instance()->root()),'\/\\'),//网址根目录
'root_website'=>(Request::instance()->isSsl()?'https':'http').'://'.trim($_SERVER['HTTP_HOST'],'\/\\').rtrim(preg_replace('/\/index\.php.*/i','',Request::instance()->root()),'\/\\'),//带域名网站根目录，去掉index.php，结尾不带/ 'root_website'=>(Request::instance()->isSsl()?'https':'http').'://'.trim(Request::instance()->host(),'\/\\').rtrim(preg_replace('/\/index\.php.*/i','',Request::instance()->root()),'\/\\'),//带域名网站根目录，去掉index.php，结尾不带/
'allow_coll_modules'=>array('pattern'),//允许的采集器模块 'allow_coll_modules'=>array('pattern'),//允许的采集器模块
'release_modules'=>array('cms','db','file','api','diy'),//发布模块 'release_modules'=>array('cms','db','file','api','diy'),//发布模块
'yzm_expire'=>1200, //邮箱验证码过期时间(秒) 'yzm_expire'=>1200, //邮箱验证码过期时间(秒)

45
SkycaijiApp/extend/util/Log.php Normal file
View File

@ -0,0 +1,45 @@
<?php
/*
|--------------------------------------------------------------------------
| SkyCaiji (蓝天采集器)
|--------------------------------------------------------------------------
| Copyright (c) 2018 http://www.skycaiji.com All rights reserved.
|--------------------------------------------------------------------------
| 使用协议 http://www.skycaiji.com/licenses
|--------------------------------------------------------------------------
*/
/*日志驱动*/
namespace util;
use think\log\driver\File;
class Log extends File{
public function save(array $log = [], $append = false){
static $passList=array(
'未定义','Undefined',
'A session had already been started','DOMDocument::loadHTML',
'MySQL server has gone away',"Error reading result set's header",
'The /e modifier is deprecated'
);
foreach ($log as $type => $val) {
foreach ($val as $key=>$msg) {
if (!is_string($msg)) {
$msg = var_export($msg, true);
}
foreach ($passList as $passStr){
if(stripos($msg, $passStr)!==false){
unset($val[$key]);
break;
}
}
}
$log[$type]=$val;
}
parent::save($log,$append);
}
}
?>

2
SkycaijiApp/install/data/check_db
View File

File diff suppressed because one or more lines are too long

2
SkycaijiApp/install/data/check_file
View File

File diff suppressed because one or more lines are too long

7
caiji Normal file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env php
<?php
// 定义项目路径
define('APP_PATH', __DIR__ . '/SkycaijiApp/');
define('APP_NAMESPACE', 'skycaiji');
// 加载框架引导文件
require __DIR__.'/thinkphp/console.php';

1
plugin/release/cms/BaseCms.php
View File

@ -53,6 +53,7 @@ abstract class BaseCms extends \skycaiji\admin\event\ReleaseBase{
} }
$cmsDb['db_type']=empty($cmsDb['db_type'])?'mysql':strtolower($cmsDb['db_type']); $cmsDb['db_type']=empty($cmsDb['db_type'])?'mysql':strtolower($cmsDb['db_type']);
$cmsDb['db_port']=$cmsDb['db_port']>0?$cmsDb['db_port']:3306;//设置默认端口 $cmsDb['db_port']=$cmsDb['db_port']>0?$cmsDb['db_port']:3306;//设置默认端口
$cmsDb['fields_strict']=false;//允许字段不存在
$this->cmsDb=$cmsDb;//cms程序配置 $this->cmsDb=$cmsDb;//cms程序配置
//实例化数据库 //实例化数据库

5
plugin/release/diy/BaseDiy.php
View File

@ -30,6 +30,11 @@ abstract class BaseDiy extends \skycaiji\admin\event\ReleaseBase{
if(empty($this->connection)){ if(empty($this->connection)){
exception('发布错误：没有数据库配置'); exception('发布错误：没有数据库配置');
} }
if(!isset($this->connection['fields_strict'])){
//默认允许字段不存在
$this->connection['fields_strict']=false;
}
//实例化数据库 //实例化数据库
try { try {
$mdb=new DbCommon($this->connection); $mdb=new DbCommon($this->connection);

5
public/static/css/admin.css
View File

@ -95,6 +95,7 @@
.datatable .desc{max-width:400px;white-space:normal;overflow:hidden;word-wrap:break-word;word-break:break-all;} .datatable .desc{max-width:400px;white-space:normal;overflow:hidden;word-wrap:break-word;word-break:break-all;}
.datatable .sort input{height:auto;padding:0 5px;} .datatable .sort input{height:auto;padding:0 5px;}
.datatable .bg0{background:#fcfcfc} .datatable .bg0{background:#fcfcfc}
.datatable .sep{color:#999;font-size:90%;}
.datatable tr,.datatable td{white-space:nowrap;} .datatable tr,.datatable td{white-space:nowrap;}
.datatable .store-info .has-update{display:inline;color:green;margin-left:5px;} .datatable .store-info .has-update{display:inline;color:green;margin-left:5px;}
@ -325,6 +326,10 @@ table.datatable thead .sorting_desc:after {
.proxy-ip-list .p-ip .form-control{min-width:150px;} .proxy-ip-list .p-ip .form-control{min-width:150px;}
.proxy-ip-list .p-ip-info{vertical-align:middle;color:#999;} .proxy-ip-list .p-ip-info{vertical-align:middle;color:#999;}
/**/ /**/
.table-test-loop thead{background:#F5F5F5;}
.table-test-loop input{border:0;overflow:hidden;}
.skin-black .nav-tabs-color li.active,.skin-black-light .nav-tabs-color li.active{border-top-color:#5A6E82!important;} .skin-black .nav-tabs-color li.active,.skin-black-light .nav-tabs-color li.active{border-top-color:#5A6E82!important;}

10
public/static/js/admin.js
View File

File diff suppressed because one or more lines are too long

2
public/static/js/admin/collected.js
View File

@ -8,5 +8,5 @@
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
*/ */
'use strict';function CollectedClass(){} 'use strict';function CollectedClass(){}
CollectedClass.prototype={constructor:CollectedClass,init_list:function(){$('#list_table .delete').bind('click',function(){var obj=$(this);confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"GET",url:obj.attr('url'),dataType:"json",success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);if(data.code==1){obj.parents('tr').eq(0).remove()}}})})});$('#deleteall').bind('click',function(){var obj=$(this);confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"POST",url:ulink('Collected/op?op=deleteall'),dataType:"json",data:$('#form_list').serialize(),success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);setTimeout("window.location.reload();",2500)}})})})}} CollectedClass.prototype={constructor:CollectedClass,init_list:function(){$('#list_table .delete').bind('click',function(){var obj=$(this);confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"GET",url:obj.attr('url'),dataType:"json",success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);if(data.code==1){obj.parents('tr').eq(0).remove()}}})})});$('#deleteall').bind('click',function(){var obj=$(this);confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"POST",url:ulink('Collected/op?op=deleteall'),dataType:"json",data:$('#form_list').serialize(),success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);setTimeout("window.location.reload();",2500)}})})});$('#btn_clear_error').bind('click',function(){windowModal('清理失败的数据',ulink('Collected/clearError'))})}}
var collectedClass=new CollectedClass() var collectedClass=new CollectedClass()

5
public/static/js/admin/collector.js
View File

@ -36,15 +36,16 @@ $(this.formid+' [name="config[paging][max]"]').val(parseInt(config.paging.max));
if(config.paging.area_module){$(this.formid+' select[name="config[paging][area_module]"]').val(config.paging.area_module).trigger('click')} if(config.paging.area_module){$(this.formid+' select[name="config[paging][area_module]"]').val(config.paging.area_module).trigger('click')}
if(config.paging.url_rule_module){$(this.formid+' select[name="config[paging][url_rule_module]"]').val(config.paging.url_rule_module).trigger('click')}} if(config.paging.url_rule_module){$(this.formid+' select[name="config[paging][url_rule_module]"]').val(config.paging.url_rule_module).trigger('click')}}
if(config.paging_fields){for(var i in config.paging_fields){$_o.paging_field_op('add',{paging_field:config.paging_fields[i]})}}} if(config.paging_fields){for(var i in config.paging_fields){$_o.paging_field_op('add',{paging_field:config.paging_fields[i]})}}}
$(document).ready(function(){var tab_link_val=$($_o.formid+' [name="tab_link"]').val();if(tab_link_val){tab_link_val='#'+tab_link_val.replace(/^#+/,'');$($_o.formid+' .coll-tab a[data-toggle="tab"]').each(function(){if(tab_link_val==$(this).attr('href')){$(this).tab('show');return}})}})},source_op:function(op,params){var $_o=this;params=params?params:{};var formObj=params.formObj?params.formObj:'#form_source';if(op=='init'){$(formObj).find('.nav-tabs li a').bind('click',function(){$(formObj).find('input[name="source[type]"]').val($(this).attr('source-type'))});$(formObj).find('div[source-param]').find('input,textarea').bind('change',function(){if($(this).attr("type")=='radio'){return!1} $(document).ready(function(){var tab_link_val=$($_o.formid+' [name="tab_link"]').val();if(tab_link_val){tab_link_val='#'+tab_link_val.replace(/^#+/,'');$($_o.formid+' .coll-tab a[data-toggle="tab"]').each(function(){if(tab_link_val==$(this).attr('href')){$(this).trigger('click');return}})}})},source_op:function(op,params){var $_o=this;params=params?params:{};var formObj=params.formObj?params.formObj:'#form_source';if(op=='init'){$(formObj).find('.nav-tabs li a').bind('click',function(){$(formObj).find('input[name="source[type]"]').val($(this).attr('source-type'))});$(formObj).find('div[source-param]').find('input,textarea').bind('change',function(){if($(this).attr("type")=='radio'){return!1}
$(this).parents('div[source-param]').find('input[name="source[param]"]').prop('checked',!0)});if(params.source){$(formObj+' .nav-tabs').find('a[source-type="'+params.source.type+'"]').click();if(params.source.type=='custom'){$(formObj+' textarea[name="source[urls]"]').val(params.source.urls)}else if(params.source.type=='batch'){$(formObj+' input[name="source[url]"]').val(params.source.url);var param_type=params.source.param;if(param_type=='num'){$(formObj+' input[name="source[param_num_start]"]').val(params.source.param_num_start);$(formObj+' input[name="source[param_num_end]"]').val(params.source.param_num_end);$(formObj+' input[name="source[param_num_inc]"]').val(params.source.param_num_inc);if(params.source.param_num_desc){$(formObj+' input[name="source[param_num_desc]"]').attr('checked','checked')}}else if(param_type=='letter'){$(formObj+' input[name="source[param_letter_start]"]').val(params.source.param_letter_start);$(formObj+' input[name="source[param_letter_end]"]').val(params.source.param_letter_end);if(params.source.param_letter_desc){$(formObj+' input[name="source[param_letter_desc]"]').attr('checked','checked')}}else if(param_type=='custom'){$(formObj+' textarea[name="source[param_custom]"]').val(params.source.param_custom)} $(this).parents('div[source-param]').find('input[name="source[param]"]').prop('checked',!0)});if(params.source){$(formObj+' .nav-tabs').find('a[source-type="'+params.source.type+'"]').click();if(params.source.type=='custom'){$(formObj+' textarea[name="source[urls]"]').val(params.source.urls)}else if(params.source.type=='batch'){$(formObj+' input[name="source[url]"]').val(params.source.url);var param_type=params.source.param;if(param_type=='num'){$(formObj+' input[name="source[param_num_start]"]').val(params.source.param_num_start);$(formObj+' input[name="source[param_num_end]"]').val(params.source.param_num_end);$(formObj+' input[name="source[param_num_inc]"]').val(params.source.param_num_inc);if(params.source.param_num_desc){$(formObj+' input[name="source[param_num_desc]"]').attr('checked','checked')}}else if(param_type=='letter'){$(formObj+' input[name="source[param_letter_start]"]').val(params.source.param_letter_start);$(formObj+' input[name="source[param_letter_end]"]').val(params.source.param_letter_end);if(params.source.param_letter_desc){$(formObj+' input[name="source[param_letter_desc]"]').attr('checked','checked')}}else if(param_type=='custom'){$(formObj+' textarea[name="source[param_custom]"]').val(params.source.param_custom)}
$(formObj+' input[name="source[param]"][value="'+param_type+'"]').attr('checked','checked')}else if(params.source.type=='large'){$(formObj+' textarea[name="source[large_urls]"]').val(params.source.large_urls)}}}else if(op=='add'){var html='';if(params.html){html=params.html}else{var html='<div class="form-group"><div class="input-group">';var regLarge=/[\r\n]/;if(regLarge.test(params.url)){html+='<textarea class="form-control" name="config[source_url][]" rows="5" id="url_'+generateUUID()+'">'+htmlspecialchars(params.url)+'</textarea>'}else{html+='<input type="text" class="form-control" name="config[source_url][]" id="url_'+generateUUID()+'" value="'+htmlspecialchars(params.url)+'">'} $(formObj+' input[name="source[param]"][value="'+param_type+'"]').attr('checked','checked')}else if(params.source.type=='large'){$(formObj+' textarea[name="source[large_urls]"]').val(params.source.large_urls)}else if(params.source.type=='api'){$(formObj+' input[name="source[api]"]').val(params.source.api);$(formObj+' input[name="source[api_json]"]').val(params.source.api_json)}}}else if(op=='add'){var html='';if(params.html){html=params.html}else{var html='<div class="form-group"><div class="input-group">';var regLarge=/[\r\n]/;if(regLarge.test(params.url)){html+='<textarea class="form-control" name="config[source_url][]" rows="5" id="url_'+generateUUID()+'">'+htmlspecialchars(params.url)+'</textarea>'}else{html+='<input type="text" class="form-control" name="config[source_url][]" id="url_'+generateUUID()+'" value="'+htmlspecialchars(params.url)+'">'}
html=html+'<div class="input-group-addon brl_0"><a class="glyphicon glyphicon-edit"></a></div>'+'<div class="input-group-addon brl_0"><a class="glyphicon glyphicon-remove"></a></div>'+'<div class="input-group-addon"><a href="javascript:;" class="glyphicon glyphicon-arrow-up"></a> <a href="javascript:;" class="glyphicon glyphicon-arrow-down"></a></div></div></div>'} html=html+'<div class="input-group-addon brl_0"><a class="glyphicon glyphicon-edit"></a></div>'+'<div class="input-group-addon brl_0"><a class="glyphicon glyphicon-remove"></a></div>'+'<div class="input-group-addon"><a href="javascript:;" class="glyphicon glyphicon-arrow-up"></a> <a href="javascript:;" class="glyphicon glyphicon-arrow-down"></a></div></div></div>'}
if(params.get){return html}else{this.source_op('clear_null');$($_o.formid+' #coll_pattern_source .c-p-source-urls').append(html)}}else if(op=='add_sub'){$.ajax({type:'POST',dataType:'json',url:$(formObj).attr('action'),data:$(formObj).serialize(),success:function(data){data.data=data.data?data.data:{};if(data.code==1){var source_type=$(formObj).find('input[name="source[type]"]').val();if(source_type=='custom'){$('#myModal').modal('hide');var urls=data.data.urls;var ix=0;var url_html_list='';for(var i in urls){ix++;if(ix==1){if(data.data.uid){$('#'+data.data.uid).val(urls[i]);continue}} if(params.get){return html}else{this.source_op('clear_null');$($_o.formid+' #coll_pattern_source .c-p-source-urls').append(html)}}else if(op=='add_sub'){$.ajax({type:'POST',dataType:'json',url:$(formObj).attr('action'),data:$(formObj).serialize(),success:function(data){data.data=data.data?data.data:{};if(data.code==1){var source_type=$(formObj).find('input[name="source[type]"]').val();if(source_type=='custom'){$('#myModal').modal('hide');var urls=data.data.urls;var ix=0;var url_html_list='';for(var i in urls){ix++;if(ix==1){if(data.data.uid){$('#'+data.data.uid).val(urls[i]);continue}}
url_html_list+=$_o.source_op('add',{get:1,url:urls[i]})} url_html_list+=$_o.source_op('add',{get:1,url:urls[i]})}
if(url_html_list){$_o.source_op('add',{html:url_html_list})}}else if(source_type=='batch'){if(params.preview==1){var urls=data.data.urls;var txt='';for(var i in urls){txt+=urls[i]+"\r\n"} if(url_html_list){$_o.source_op('add',{html:url_html_list})}}else if(source_type=='batch'){if(params.preview==1){var urls=data.data.urls;var txt='';for(var i in urls){txt+=urls[i]+"\r\n"}
$(formObj).find('#source_preview').val(txt)}else{if(data.data.uid){$('#'+data.data.uid).val(data.data.url)}else{$_o.source_op('add',{url:data.data.url})} $(formObj).find('#source_preview').val(txt)}else{if(data.data.uid){$('#'+data.data.uid).val(data.data.url)}else{$_o.source_op('add',{url:data.data.url})}
$('#myModal').modal('hide')}}else if(source_type=='large'){var large_urls=data.data.urls;large_urls=large_urls.join("\r\n");if(data.data.uid){var cur_ele=$('#'+data.data.uid).parents('.form-group').eq(0);var large_ele=$_o.source_op('add',{get:1,url:large_urls});$(cur_ele).replaceWith(large_ele)}else{$_o.source_op('add',{url:large_urls})} $('#myModal').modal('hide')}}else if(source_type=='large'){var large_urls=data.data.urls;large_urls=large_urls.join("\r\n");if(data.data.uid){var cur_ele=$('#'+data.data.uid).parents('.form-group').eq(0);var large_ele=$_o.source_op('add',{get:1,url:large_urls});$(cur_ele).replaceWith(large_ele)}else{$_o.source_op('add',{url:large_urls})}
$('#myModal').modal('hide')}else if(source_type=='api'){if(data.data.uid){$('#'+data.data.uid).val(data.data.url)}else{$_o.source_op('add',{url:data.data.url})}
$('#myModal').modal('hide')}}else{toastr.error(data.msg)}},error:function(data){toastr.error(data)}});return!1}else if(op=='clear_null'){$($_o.formid+' #coll_pattern_source').find('[name="config[source_url][]"]').each(function(){if(!$(this).val()){$(this).parents('.form-group').eq(0).remove()}})}else if(op=='clear_all'){confirmRight('是否清空网址？',function(){$($_o.formid+' #coll_pattern_source').find('[name="config[source_url][]"]').each(function(){$(this).parents('.form-group').eq(0).remove()})})}},field_op:function(op,params){var $_o=this;params=params?params:{};var formObj=params.formObj?params.formObj:'#form_field';if(op=='init'){var level_urls=new Array();$('#c_p_level_urls [id^="level_url_"]').each(function(){var levelName=$(this).find('.name').text();level_urls.push('<option value="level_url:'+levelName+'">'+levelName+'</option>')});if(($($_o.formid+' [name="config[source_is_url]"]').is(':checked')!=!0)&&level_urls.length>0){$(formObj+' select[name="field[source]"]').append('<optgroup label="多级页">'+level_urls.join('')+'</optgroup>')} $('#myModal').modal('hide')}}else{toastr.error(data.msg)}},error:function(data){toastr.error(data)}});return!1}else if(op=='clear_null'){$($_o.formid+' #coll_pattern_source').find('[name="config[source_url][]"]').each(function(){if(!$(this).val()){$(this).parents('.form-group').eq(0).remove()}})}else if(op=='clear_all'){confirmRight('是否清空网址？',function(){$($_o.formid+' #coll_pattern_source').find('[name="config[source_url][]"]').each(function(){$(this).parents('.form-group').eq(0).remove()})})}},field_op:function(op,params){var $_o=this;params=params?params:{};var formObj=params.formObj?params.formObj:'#form_field';if(op=='init'){var level_urls=new Array();$('#c_p_level_urls [id^="level_url_"]').each(function(){var levelName=$(this).find('.name').text();level_urls.push('<option value="level_url:'+levelName+'">'+levelName+'</option>')});if(($($_o.formid+' [name="config[source_is_url]"]').is(':checked')!=!0)&&level_urls.length>0){$(formObj+' select[name="field[source]"]').append('<optgroup label="多级页">'+level_urls.join('')+'</optgroup>')}
var relation_urls=new Array();$('#c_p_relation_urls [id^="relation_url_"]').each(function(){var relationName=$(this).find('.name').text();relation_urls.push('<option value="relation_url:'+relationName+'">'+relationName+'</option>')});if(relation_urls.length>0){$(formObj+' select[name="field[source]"]').append('<optgroup label="关联页">'+relation_urls.join('')+'</optgroup>')} var relation_urls=new Array();$('#c_p_relation_urls [id^="relation_url_"]').each(function(){var relationName=$(this).find('.name').text();relation_urls.push('<option value="relation_url:'+relationName+'">'+relationName+'</option>')});if(relation_urls.length>0){$(formObj+' select[name="field[source]"]').append('<optgroup label="关联页">'+relation_urls.join('')+'</optgroup>')}
$(formObj+' select[name="field[module]"]').bind('change',function(){$(formObj+' .c-p-field-module').hide();$(formObj+' .c-p-field-module[module="'+$(this).val()+'"]').show();var source_module=new Array('rule','xpath','json','auto');var in_module=!1;for(var i in source_module){if($(this).val()==source_module[i]){in_module=!0;break}} $(formObj+' select[name="field[module]"]').bind('change',function(){$(formObj+' .c-p-field-module').hide();$(formObj+' .c-p-field-module[module="'+$(this).val()+'"]').show();var source_module=new Array('rule','xpath','json','auto');var in_module=!1;for(var i in source_module){if($(this).val()==source_module[i]){in_module=!0;break}}

2
public/static/js/admin/cpattern_elements.js
View File

@ -20,4 +20,4 @@ xpath=xpath.replace(/\/\//g,' ');xpath=xpath.replace(/\//g,'>');xpath=xpath.repl
var $_o=this;xpath=xpath.split('/');var reg=/^((?!(html|body))\w)+$/;var ix=-1;for(var i=(xpath.length-1);i>=0;i--){if(reg.test(xpath[i])){ix=i;break}} var $_o=this;xpath=xpath.split('/');var reg=/^((?!(html|body))\w)+$/;var ix=-1;for(var i=(xpath.length-1);i>=0;i--){if(reg.test(xpath[i])){ix=i;break}}
var isShow=$(obj).text()=='显示'?true:!1;var bgColor=(isShow?'#C8ECE6':'');if(ix>-1){var parentXpath=xpath.slice(0,ix+1);parentXpath=parentXpath.join('/');var subXpath=xpath.slice(ix+1);subXpath=subXpath.join('/');var parentCsspath=$_o.xpath2csspath(parentXpath);var subCsspath=$_o.xpath2csspath(subXpath);if(subCsspath){var curIndex=-1;$(parentCsspath).each(function(){curIndex++;var curCsspath=parentCsspath+':eq('+curIndex+')>'+subCsspath;$(curCsspath).css({'background-color':bgColor})})}else{$(parentCsspath).css({'background-color':bgColor})}}else{var csspath=$_o.xpath2csspath(xpath.join('/'));$(csspath).css({'background-color':bgColor})} var isShow=$(obj).text()=='显示'?true:!1;var bgColor=(isShow?'#C8ECE6':'');if(ix>-1){var parentXpath=xpath.slice(0,ix+1);parentXpath=parentXpath.join('/');var subXpath=xpath.slice(ix+1);subXpath=subXpath.join('/');var parentCsspath=$_o.xpath2csspath(parentXpath);var subCsspath=$_o.xpath2csspath(subXpath);if(subCsspath){var curIndex=-1;$(parentCsspath).each(function(){curIndex++;var curCsspath=parentCsspath+':eq('+curIndex+')>'+subCsspath;$(curCsspath).css({'background-color':bgColor})})}else{$(parentCsspath).css({'background-color':bgColor})}}else{var csspath=$_o.xpath2csspath(xpath.join('/'));$(csspath).css({'background-color':bgColor})}
$(obj).text(isShow?'取消':'显示')},close:function(){$('#skycaiji_console').remove();$('*').css({'background-color':''})}} $(obj).text(isShow?'取消':'显示')},close:function(){$('#skycaiji_console').remove();$('*').css({'background-color':''})}}
var skycaijiCE=null;$(document).ready(function(){skycaijiCE=new SkycaijiCpatternElement();skycaijiCE.init()}) var skycaijiCE=new SkycaijiCpatternElement()

17
public/static/js/admin/json_tree.js Normal file
View File

@ -0,0 +1,17 @@
/*
|--------------------------------------------------------------------------
| SkyCaiji (蓝天采集器)
|--------------------------------------------------------------------------
| Copyright (c) 2018 http://www.skycaiji.com All rights reserved.
|--------------------------------------------------------------------------
| 使用协议 http://www.skycaiji.com/licenses
|--------------------------------------------------------------------------
*/
'use strict';function JsonTree(){this.treeId='';this.treeClass=''}
JsonTree.prototype={constructor:JsonTree,load:function(data){var $_o=this;if(htmlIsJson(data)){data=JSON.parse(data);if(data){var html='<ul>';for(var i in data){html+=$_o.node(i,data[i])}
html+='</ul>';$($_o.treeId).html(html)}}
$($_o.treeId).on('click',$_o.treeClass,function(){if($(this).hasClass('glyphicon-triangle-bottom')){$(this).removeClass('glyphicon-triangle-bottom');$(this).addClass('glyphicon-triangle-right');var hasSub=!1;$(this).siblings('ul').children('li').each(function(){var subTree=$(this).find($_o.treeClass).eq(0);if(subTree.length>0){subTree.removeClass('glyphicon-triangle-bottom').addClass('glyphicon-triangle-right').siblings('ul').hide();hasSub=!0}});if(!hasSub){$(this).siblings('ul').hide()}}else{$(this).removeClass('glyphicon-triangle-right');$(this).addClass('glyphicon-triangle-bottom');$(this).siblings('ul').show();$(this).siblings('ul').children('li').each(function(){$(this).find($_o.treeClass).eq(0).removeClass('glyphicon-triangle-right').addClass('glyphicon-triangle-bottom').siblings('ul').show()})}})},node:function(node,list){var $_o=this;var html='<li>';var isList=!1;if(list){if(typeof(list)=='object'&&!$.isEmptyObject(list)){isList=!0}}
if(isList){html+='<span class="glyphicon glyphicon-triangle-bottom tree"></span><span class="node">'+node+'</span><ul>';for(var i in list){html+=$_o.node(i,list[i])}
html+='</ul>'}else{html+='<span class="node">'+node+'</span>: <span class="val">'+list+'</span>'}
html+='</li>';return html}}
var jsonTree=new JsonTree()

12
public/static/js/admin/task.js
View File

@ -7,10 +7,14 @@
| 使用协议 http://www.skycaiji.com/licenses | 使用协议 http://www.skycaiji.com/licenses
|-------------------------------------------------------------------------- |--------------------------------------------------------------------------
*/ */
$(document).ready(function(){$('#task_folder').on('click','.taskgroup',function(){var trBox=$(this).parents('tr').eq(0);var tgid=trBox.attr('tgid');var tgLevel=trBox.attr('level');var url=ulink('Task/openList?tg_id=_tgid_');var loadingImg=tgLevel>0?'<img src="'+window.site_config.pub+'/static/images/load1.gif" class="img_loading" style="width:14px;">':'<img src="'+window.site_config.pub+'/static/images/loading.gif"> '+window.tpl_lang.task_loading;url=url.replace('_tgid_',tgid);var $_o=$(this);var opened=trBox.attr('opened');var childs=$('#task_folder tr[parent-tgid="'+tgid+'"]');childs.css('display',(1==opened?'none':'table-row'));childs.each(function(){var cTgid=$(this).attr('tgid');var cOpened=$(this).attr('opened');if(cTgid){if(1==opened){$('#task_folder tr[parent-tgid="'+cTgid+'"]').css('display','none')}else{$('#task_folder tr[parent-tgid="'+cTgid+'"]').css('display',1==cOpened?'table-row':'none')}}});if(1==opened){trBox.attr('opened',0);$_o.find('.glyphicon-minus').addClass('glyphicon-plus').removeClass('glyphicon-minus')}else{trBox.attr('opened',1);$_o.find('.glyphicon-plus').addClass('glyphicon-minus').removeClass('glyphicon-plus');if(trBox.attr('setted')!=1){$_o.parents('td').eq(0).append(loadingImg);$.ajax({type:"GET",url:url,dataType:"json",success:function(data){if(tgLevel>0){$_o.siblings('.img_loading').remove()}else{trBox.hide()} 'use strict';$(document).ready(function(){$('#task_folder').on('click','.taskgroup',function(){var trBox=$(this).parents('tr').eq(0);var tgid=trBox.attr('tgid');var tgLevel=trBox.attr('level');var url=ulink('Task/openList?tg_id=_tgid_');var loadingImg=tgLevel>0?'<img src="'+window.site_config.pub+'/static/images/load1.gif" class="img_loading" style="width:14px;">':'<img src="'+window.site_config.pub+'/static/images/loading.gif"> '+window.tpl_lang.task_loading;url=url.replace('_tgid_',tgid);var $_o=$(this);var opened=trBox.attr('opened');var childs=$('#task_folder tr[parent-tgid="'+tgid+'"]');childs.css('display',(1==opened?'none':'table-row'));childs.each(function(){var cTgid=$(this).attr('tgid');var cOpened=$(this).attr('opened');if(cTgid){if(1==opened){$('#task_folder tr[parent-tgid="'+cTgid+'"]').css('display','none')}else{$('#task_folder tr[parent-tgid="'+cTgid+'"]').css('display',1==cOpened?'table-row':'none')}}});if(1==opened){trBox.attr('opened',0);$_o.find('.glyphicon-minus').addClass('glyphicon-plus').removeClass('glyphicon-minus')}else{trBox.attr('opened',1);$_o.find('.glyphicon-plus').addClass('glyphicon-minus').removeClass('glyphicon-plus');if(trBox.attr('setted')!=1){$_o.parents('td').eq(0).append(loadingImg);$.ajax({type:"GET",url:url,dataType:"json",success:function(data){if(tgLevel>0){$_o.siblings('.img_loading').remove()}else{trBox.hide()}
tgLevel=parseInt(tgLevel);var leftSpace=tgLevel>0?('padding-left:'+tgLevel*25+'px;'):'';var html='';if(data.code==1){if(isNull(data.data)){html=null}else{var html='';if(data.data.tgList){var datalist=data.data.tgList;for(var i in datalist){html+='<tr level="'+(tgLevel+1)+'" parent-tgid="'+tgid+'" tgid="'+datalist[i].id+'"><td style="'+leftSpace+'"><a href="javascript:;" class="taskgroup"> <span class="glyphicon glyphicon-plus"></span> <span class="glyphicon glyphicon-folder-close"></span> '+datalist[i].name+' </a></td><td colspan="7"></td></tr>'}} tgLevel=parseInt(tgLevel);var leftSpace=tgLevel>0?('padding-left:'+tgLevel*25+'px;'):'';var html='';if(data.code==1){if(isNull(data.data)){html=null}else{var html='';if(data.data.tgList){var datalist=data.data.tgList;for(var i in datalist){html+='<tr level="'+(tgLevel+1)+'" parent-tgid="'+tgid+'" tgid="'+datalist[i].id+'"><td style="'+leftSpace+'"><a href="javascript:;" class="taskgroup"> <span class="glyphicon glyphicon-plus"></span> <span class="glyphicon glyphicon-folder-close"></span> '+datalist[i].name+' </a></td><td colspan="7"></td></tr>'}}
if(data.data.taskList){var datalist=data.data.taskList;for(var i in datalist){html+='<tr parent-tgid="'+tgid+'" task-id="'+datalist[i].id+'"></td><td style="'+leftSpace+'">'+'<a href="'+ulink('Task/edit?id='+datalist[i].id)+'" class="edit">'+datalist[i].name+'</a></td><td class="sort"><input type="text" name="newsort['+datalist[i].id+']" class="form-control" value="'+datalist[i].sort+'" autocomplete="off" /></td>'+'<td><a href="javascript:;" class="auto" item-id="'+datalist[i].id+'" style="color:'+(datalist[i].auto>0?'green':'red')+';">'+(datalist[i].auto>0?window.tpl_lang.yes:window.tpl_lang.no)+'</a></td>'+'<td>'+datalist[i].module+'</td><td>'+datalist[i].addtime+'</td><td>'+'<label class="checkbox-inline"><input type="checkbox" name="batch[]" value="'+datalist[i].id+'" /> '+datalist[i].caijitime+'</label>'+' &nbsp;<a href="javascript:;" class="caiji" item-id="'+datalist[i].id+'">'+window.tpl_lang.caiji+'</a></td>'+'<td><a href="'+ulink('Collector/set?task_id='+datalist[i].id)+'">规则</a>'+' &nbsp;<a href="'+ulink('Release/set?task_id='+datalist[i].id)+'">发布</a>'+' &nbsp;<a href="javascript:;" class="delete" item-id="'+datalist[i].id+'">'+window.tpl_lang.delete+'</a></td></tr>'}}}}else{html=null} if(data.data.taskList){var datalist=data.data.taskList;for(var i in datalist){html+='<tr parent-tgid="'+tgid+'" task-id="'+datalist[i].id+'"></td><td style="'+leftSpace+'">'+'<a href="'+ulink('Task/edit?id='+datalist[i].id)+'" class="edit">'+datalist[i].name+'</a></td><td class="sort"><input type="text" name="newsort['+datalist[i].id+']" class="form-control" value="'+datalist[i].sort+'" autocomplete="off" /></td>'+'<td><a href="javascript:;" class="auto" item-id="'+datalist[i].id+'" style="color:'+(datalist[i].auto>0?'green':'red')+';">'+(datalist[i].auto>0?window.tpl_lang.yes:window.tpl_lang.no)+'</a></td>'+'<td>'+datalist[i].module+'</td><td>'+datalist[i].addtime+'</td><td>'+'<label class="checkbox-inline"><input type="checkbox" name="batch[]" value="'+datalist[i].id+'" /> '+datalist[i].caijitime+'</label>'+' &nbsp;<a href="javascript:;" class="caiji" item-id="'+datalist[i].id+'">'+window.tpl_lang.caiji+'</a>'+' <span class="sep">|</span> <a href="javascript:;" class="houtai" item-id="'+datalist[i].id+'">后台</a>'+'</td><td><a href="'+ulink('Collector/set?task_id='+datalist[i].id)+'">规则</a>'+' <span class="sep">|</span> <a href="'+ulink('Release/set?task_id='+datalist[i].id)+'">发布</a>'+' <span class="sep">|</span> <a href="javascript:;" class="delete" item-id="'+datalist[i].id+'">'+window.tpl_lang.delete+'</a></td></tr>'}}}}else{html=null}
html=isNull(html)?('<tr parent-tgid="'+tgid+'"><td style="'+leftSpace+'">'+window.tpl_lang.task_none_data+'</td><td colspan="6"></td></tr>'):html;trBox.attr('setted',1);trBox.after(html)}})}}});$('#task_folder #level0 .taskgroup').trigger('click')});$(document).ready(function(){$('table.datatable').on('click','.delete',function(){var obj=$(this);var url=ulink('Task/op?op=delete&id=_id_',{_id_:obj.attr('item-id')});confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"GET",url:url,dataType:"json",success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);if(data.code==1){obj.parents('tr').eq(0).remove()}}})})});$('table.datatable').on('click','.auto',function(){var auto=1;var tips=[window.tpl_lang.yes,'green'];if($(this).html()==window.tpl_lang.yes){auto=0;tips=[window.tpl_lang.no,'red']} html=isNull(html)?('<tr parent-tgid="'+tgid+'"><td style="'+leftSpace+'">'+window.tpl_lang.task_none_data+'</td><td colspan="6"></td></tr>'):html;trBox.attr('setted',1);trBox.after(html)}})}}});$('#task_folder #level0 .taskgroup').trigger('click')});var click_task_backstage_time={};function task_backstage_collect(ids){var isBatch=!1;if($.isArray(ids)){isBatch=!0;ids=ids.join(',')}
var itemid=$(this).attr('item-id');if(itemid>0){var $_o=$(this);var url=ulink('Task/op?op=auto&auto=_auto_&id=_id_');url=url.replace('_auto_',auto).replace('_id_',itemid);$.ajax({type:'GET',url:url,success:function(data){if(data.code==1){$_o.html(tips[0]);$_o.css('color',tips[1])}},dataType:'json'})}});$('table.datatable').on('click','.caiji',function(){var taskid=$(this).attr('item-id');var url=ulink('Task/collect?id=_id_',{'_id_':taskid});windowIframe(window.tpl_lang.task_caiji_ing,url,{lg:1,loaded_func:function(){$('#myModal .modal-footer .close').html('关闭采集').prop('title','关闭并结束任务').addClass('btn btn-primary').removeClass('close');$('#myModal .modal-footer').prepend('<button type="button" class="btn btn-default backstage" data-dismiss="modal" title="关闭并在后台运行任务">后台运行</button>');$('#myModal .modal-footer .backstage').bind('click',function(){$.ajax({type:'GET',url:ulink('Task/collect?id=_id_&backstage=1',{'_id_':taskid}),async:!0,timeout:1000,dataType:'html'})})}})});$('table.datatable').on('click','.caiji-batch',function(){var taskids=[];$('input[name="batch[]"]:checked').each(function(){taskids.push($(this).val())});taskids=taskids.join(',');var url=ulink('Task/collectBatch?ids=_ids_',{'_ids_':taskids});windowIframe(window.tpl_lang.task_caiji_ing,url,{lg:1,loaded_func:function(){$('#myModal .modal-footer .close').html('关闭采集').prop('title','关闭并结束任务').addClass('btn btn-primary').removeClass('close');$('#myModal .modal-footer').prepend('<button type="button" class="btn btn-default backstage" data-dismiss="modal" title="关闭并在后台运行任务">后台运行</button>');$('#myModal .modal-footer .backstage').bind('click',function(){$.ajax({type:'GET',url:ulink('Task/collectBatch?ids=_ids_&backstage=1',{'_ids_':taskids}),async:!0,timeout:1000,dataType:'html'})})}})});$('table.datatable thead th[data-order]').bind('click',function(){var order=$(this).attr('data-order');if(!order){return!1} if(!click_task_backstage_time[ids]){click_task_backstage_time[ids]=(new Date()).getTime()}else{if((new Date()).getTime()-click_task_backstage_time[ids]<=3000){return!1}else{click_task_backstage_time[ids]=(new Date()).getTime()}}
var url='';if(isBatch){url=ulink('Task/collectBatch?ids=_ids_&backstage=1',{'_ids_':ids})}else{url=ulink('Task/collect?id=_id_&backstage=1',{'_id_':ids})}
$.ajax({type:'GET',url:url,async:!0,timeout:1000,dataType:'html'});toastr.success('已添加到后台运行');backstageTask()}
$(document).ready(function(){$('table.datatable').on('click','.delete',function(){var obj=$(this);var url=ulink('Task/op?op=delete&id=_id_',{_id_:obj.attr('item-id')});confirmRight(window.tpl_lang.confirm_delete,function(){$.ajax({type:"GET",url:url,dataType:"json",success:function(data){data.code==1?toastr.success(data.msg):toastr.error(data.msg);if(data.code==1){obj.parents('tr').eq(0).remove()}}})})});$('table.datatable').on('click','.auto',function(){var auto=1;var tips=[window.tpl_lang.yes,'green'];if($(this).html()==window.tpl_lang.yes){auto=0;tips=[window.tpl_lang.no,'red']}
var itemid=$(this).attr('item-id');if(itemid>0){var $_o=$(this);var url=ulink('Task/op?op=auto&auto=_auto_&id=_id_');url=url.replace('_auto_',auto).replace('_id_',itemid);$.ajax({type:'GET',url:url,success:function(data){if(data.code==1){$_o.html(tips[0]);$_o.css('color',tips[1])}},dataType:'json'})}});$('table.datatable').on('click','.caiji',function(){var taskid=$(this).attr('item-id');var url=ulink('Task/collect?id=_id_',{'_id_':taskid});windowIframe(window.tpl_lang.task_caiji_ing,url,{lg:1,loaded_func:function(){$('#myModal .modal-footer .close').html('关闭采集').prop('title','关闭并结束任务').addClass('btn btn-primary').removeClass('close');$('#myModal .modal-footer').prepend('<button type="button" class="btn btn-default backstage" data-dismiss="modal" title="关闭并在后台运行任务">后台运行</button>');$('#myModal .modal-footer .backstage').bind('click',function(){task_backstage_collect(taskid)})}})});$('table.datatable').on('click','.caiji-batch',function(){var taskids=[];$('input[name="batch[]"]:checked').each(function(){taskids.push($(this).val())});var url=ulink('Task/collectBatch?ids=_ids_',{'_ids_':taskids.join(',')});windowIframe(window.tpl_lang.task_caiji_ing,url,{lg:1,loaded_func:function(){$('#myModal .modal-footer .close').html('关闭采集').prop('title','关闭并结束任务').addClass('btn btn-primary').removeClass('close');$('#myModal .modal-footer').prepend('<button type="button" class="btn btn-default backstage" data-dismiss="modal" title="关闭并在后台运行任务">后台运行</button>');$('#myModal .modal-footer .backstage').bind('click',function(){task_backstage_collect(taskids)})}})});$('table.datatable').on('click','.houtai',function(){var taskid=$(this).attr('item-id');task_backstage_collect(taskid)});$('table.datatable').on('click','.houtai-batch',function(){var taskids=[];$('input[name="batch[]"]:checked').each(function(){taskids.push($(this).val())});task_backstage_collect(taskids)});$('table.datatable thead th[data-order]').bind('click',function(){var order=$(this).attr('data-order');if(!order){return!1}
var className=$(this).attr('class');var sort='desc';if(className=='sorting_desc'){sort='asc'} var className=$(this).attr('class');var sort='desc';if(className=='sorting_desc'){sort='asc'}
window.location.href=ulink('Task/list?show=list&order='+order+'&sort='+sort);return!1})}) window.location.href=ulink('Task/list?show=list&order='+order+'&sort='+sort);return!1})})