jinql
/
skycaiji
mirror of https://gitee.com/zorlan/skycaiji
1
0
Fork 0
Code Issues Releases Wiki Activity
skycaiji/SkycaijiApp/admin/event/Cpattern.php

1899 lines
58 KiB
PHP
Raw Normal View History

2.0
2019-02-19 09:52:13 +00:00
<?php
/*
|--------------------------------------------------------------------------
| SkyCaiji (蓝天采集器)
|--------------------------------------------------------------------------
2.2
2019-06-23 02:20:58 +00:00
| Copyright (c) 2018 https://www.skycaiji.com All rights reserved.
2.0
2019-02-19 09:52:13 +00:00
|--------------------------------------------------------------------------
2.2
2019-06-23 02:20:58 +00:00
| 使用协议 https://www.skycaiji.com/licenses
2.0
2019-02-19 09:52:13 +00:00
|--------------------------------------------------------------------------
*/
namespace skycaiji\admin\event;
use skycaiji\admin\model\CacheModel;
2.2
2019-06-23 02:20:58 +00:00
class Cpattern extends CpatternBase{
/*采集,return false表示终止采集*/
public function collect($num=10){
if(!defined('IS_COLLECTING')){
define('IS_COLLECTING', 1);
}
@session_start();
\think\Session::pause();
if(!$this->show_opened_tools){
$opened_tools=array();
if($this->config['page_render']){
$opened_tools[]='页面渲染';
}
if($GLOBALS['config']['caiji']['download_img']){
$opened_tools[]='图片本地化';
}
if($GLOBALS['config']['proxy']['open']){
$opened_tools[]='代理';
}
if(!empty($opened_tools)){
$this->echo_msg('开启功能：'.implode('、', $opened_tools),'black');
}
if($num>0){
$this->echo_msg('预计采集'.$num.'条数据','black');
}
$this->show_opened_tools=true;
}
$this->collect_num=$num;
$this->collected_field_list=array();
$source_is_url=intval($this->config['source_is_url']);
if(!isset($this->original_source_urls)){
$this->original_source_urls=array();
foreach ( $this->config ['source_url'] as $k => $v ) {
if(empty($v)){
continue;
}
$return_s_urls = $this->convert_source_url ( $v );
if (is_array ( $return_s_urls )) {
foreach ($return_s_urls as $r_s_u){
$this->original_source_urls[md5($r_s_u)]=$r_s_u;
}
} else {
$this->original_source_urls[md5($return_s_urls)]=$return_s_urls;
}
}
}
if(empty($this->original_source_urls)){
$this->echo_msg('没有起始页网址！');
return 'completed';
}
if($source_is_url){
if(isset($this->used_source_urls['_source_is_url_'])){
$this->echo_msg('所有起始页采集完毕！','green');
return 'completed';
}
}else{
if(count($this->original_source_urls)<=count($this->used_source_urls)){
$this->echo_msg('所有起始页采集完毕！','green');
return 'completed';
}
}
$source_interval=$GLOBALS['config']['caiji']['interval']*60;
$time_interval_list=array();
$source_urls=array();
$mcacheSource=CacheModel::getInstance('source_url');
if($source_is_url){
$source_urls=$this->original_source_urls;
}else{
$cacheSources=$mcacheSource->db()->where(array('cname'=>array('in',array_keys($this->original_source_urls))))->column('dateline','cname');
if(!empty($cacheSources)){
$count_db_used=0;
$sortSources=array('undb'=>array(),'db'=>array());
foreach ($this->original_source_urls as $sKey=>$sVal){
if(!isset($cacheSources[$sKey])){
$sortSources['undb'][$sKey]=$sVal;
}else{
$time_interval=abs(NOW_TIME-$cacheSources[$sKey]);
if($time_interval<$source_interval){
$this->used_source_urls[$sVal]=1;
$count_db_used++;
$time_interval_list[]=$time_interval;
}else{
$sortSources['db'][$sKey]=$sVal;
}
}
}
if($count_db_used>0){
$this->echo_msg($count_db_used.'条已采集起始网址被过滤，下次采集需等待'.($source_interval-max($time_interval_list)).'秒，<a href="'
.url('Admin/Setting/caiji').'" target="_blank">设置间隔</a>','black');
if(count($this->original_source_urls)<=count($this->used_source_urls)){
$this->echo_msg('所有起始页采集完毕！','green');
return 'completed';
}
}
$source_urls=array_merge($sortSources['undb'],$sortSources['db']);
unset($sortSources);
unset($cacheSources);
}else{
$source_urls=$this->original_source_urls;
}
}
$mcollected=model('Collected');
if($source_is_url){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$this->cont_urls_list['_source_is_url_']=array_values($source_urls);
$source_urls=array('_source_is_url_'=>'_source_is_url_');
}
foreach ($source_urls as $key_source_url=>$source_url){
$this->cur_source_url=$source_url;
if(array_key_exists($source_url,$this->used_source_urls)){
continue;
}
if($source_is_url){
$this->echo_msg("起始页已转换为内容页网址",'black');
}else{
$this->echo_msg("采集起始页：<a href='{$source_url}' target='_blank'>{$source_url}</a>",'green');
}
if($source_is_url){
$this->_collect_fields();
}else{
if(!empty($this->config['level_urls'])){
$this->echo_msg('开始分析多级网址','black');
$return_msg=$this->_collect_level($source_url,1);
if($return_msg=='completed'){
return $return_msg;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
}else{
$cont_urls=$this->getContUrls($source_url);
$this->cont_urls_list[$source_url]=$this->_collect_unused_cont_urls($cont_urls);
$this->_collect_fields();
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){
break;
}
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
return $this->collected_field_list;
2.0
2019-02-19 09:52:13 +00:00
}
/**
* 优化设置页面post过来的config
* @param unknown $config
*/
public function setConfig($config){
$config['url_complete']=intval($config['url_complete']);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$config['url_reverse']=intval($config['url_reverse']);
$config['page_render']=intval($config['page_render']);
$config['url_repeat']=intval($config['url_repeat']);
if(!empty($config['request_headers'])){
if(!is_array($config['request_headers']['custom_names'])){
$config['request_headers']['custom_names']=array();
}
if(!is_array($config['request_headers']['custom_vals'])){
$config['request_headers']['custom_vals']=array();
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
foreach ($config['request_headers']['custom_names'] as $k=>$v){
if(empty($v)){
unset($config['request_headers']['custom_names'][$k]);
unset($config['request_headers']['custom_vals'][$k]);
}
}
$config['request_headers']['custom_names']=array_values($config['request_headers']['custom_names']);
$config['request_headers']['custom_vals']=array_values($config['request_headers']['custom_vals']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
foreach ($config['source_url'] as $k=>$v){
if(preg_match('/[\r\n]/', $v)){
if(preg_match_all('/^\w+\:\/\/[^\r\n]+/im',$v,$v_urls)){
$v_urls=array_unique($v_urls[0]);
$v_urls=array_values($v_urls);
$config['source_url'][$k]=implode("\r\n", $v_urls);
}else{
unset($config['source_url'][$k]);
}
}else{
if(!preg_match('/^\w+\:\/\/.+/i', $v)){
unset($config['source_url'][$k]);
}
}
}
$config['source_url']=array_unique($config['source_url']);
$config['source_url']=array_filter($config['source_url']);
$config['source_url']=array_values($config['source_url']);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['field_list'])){
foreach ($config['field_list'] as $k=>$v){
$config['field_list'][$k]=json_decode(url_b64decode($v),true);
}
}
if(!empty($config['field_process'])){
foreach ($config['field_process'] as $k=>$v){
$config['field_process'][$k]=json_decode(url_b64decode($v),true);
$config['field_process'][$k]=$this->setProcess($config['field_process'][$k]);
}
}
$config['common_process']=input('process/a',null,'trim');
$config['common_process']=$this->setProcess($config['common_process']);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['paging_fields'])){
foreach ($config['paging_fields'] as $k=>$v){
$config['paging_fields'][$k]=json_decode(url_b64decode($v),true);
}
}
if(!empty($config['level_urls'])){
foreach ($config['level_urls'] as $k=>$v){
$config['level_urls'][$k]=json_decode(url_b64decode($v),true);
}
}
if(!empty($config['relation_urls'])){
foreach ($config['relation_urls'] as $k=>$v){
$config['relation_urls'][$k]=json_decode(url_b64decode($v),true);
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$config['url_post']=intval($config['url_post']);
if(!empty($config['url_posts'])){
if(!is_array($config['url_posts']['names'])){
$config['url_posts']['names']=array();
}
if(!is_array($config['url_posts']['vals'])){
$config['url_posts']['vals']=array();
}
if(!empty($config['url_posts']['names'])){
foreach ($config['url_posts']['names'] as $k=>$v){
if(empty($v)){
unset($config['url_posts']['names'][$k]);
unset($config['url_posts']['vals'][$k]);
}
}
}
$config['url_posts']['names']=array_values($config['url_posts']['names']);
$config['url_posts']['vals']=array_values($config['url_posts']['vals']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
return $config;
}
public function init($collData){
$collData['config']=unserialize($collData['config']);
$this->collector=$collData;
$releData=model('Release')->where(array('task_id'=>$collData['task_id']))->find();
if(!empty($releData)){
$releData=$releData->toArray();
}
$this->release=$releData;
$keyConfig='collector_config_'.$collData['id'];
$cacheConfig=cache($keyConfig);
if(empty($cacheConfig)||$cacheConfig['update_time']!=$collData['uptime']){
$config=$this->initConfig($collData['config']);
cache($keyConfig,array('update_time'=>$collData['uptime'],'config'=>$config));
}else{
$config=$cacheConfig['config'];
}
$this->config=$config;
}
public function initConfig($config){
$newConfig=array();
$newConfig['charset'] = $config['charset']=='custom' ? $config ['charset_custom'] : $config ['charset'];
$newConfig['charset']= empty($newConfig['charset'])?'auto':$newConfig['charset'];
if(!empty($config['area'])){
if(empty($config['area_module'])){
$newConfig['reg_source_cont']=$this->convert_sign_match($config['area']);
}
}elseif(!empty($config['area_start'])||!empty($config['area_end'])) {
$newConfig['reg_source_cont'] = $config['area_start'] . (!empty($config['area_end']) ? '(?P<match>[\s\S]+?)' : '(?P<match>[\s\S]+)') . $config['area_end'];
}
if(!empty($newConfig['reg_source_cont'])){
$newConfig['reg_source_cont'] = str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont'] );
$newConfig['reg_source_cont'] = preg_replace ( '/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont'] );
}elseif(!empty($config['area_module'])){
$newConfig['reg_source_cont']=$config['area'];
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(empty($config['url_rule_module'])){
if(!empty($config['url_rule'])){
$newConfig['reg_source_cont_url']=$this->convert_sign_match($config['url_rule']);
$newConfig['reg_source_cont_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont_url']);
$newConfig['reg_source_cont_url']=str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont_url'] );
}else{
$newConfig['reg_source_cont_url']='\bhref=[\'\"](?P<match>[^\'\"\<\>]+?)[\'\"]';
}
$config['url_merge']=$this->set_merge_default($newConfig['reg_source_cont_url'], $config['url_merge']);
}elseif('xpath'==$config['url_rule_module']){
if(!empty($config['url_rule'])){
$newConfig['reg_source_cont_url']=$config['url_rule'];
}else{
$newConfig['reg_source_cont_url']='//a';
}
$config['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['url_merge']);
}elseif('json'==$config['url_rule_module']){
$newConfig['reg_source_cont_url']=$config['url_rule'];
$config['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['url_merge']);
}
if(!empty($config['url_must'])){
$newConfig['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_must']);
$newConfig['url_must']=str_replace('(*)', '[\s\S]*?', $newConfig['url_must']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['url_ban'])){
$newConfig['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_ban']);
$newConfig['url_ban']=str_replace('(*)', '[\s\S]*?', $newConfig['url_ban']);
}
if(!empty($config['level_urls'])){
$config['new_level_urls']=array();
foreach ($config['level_urls'] as $luk=>$luv){
if(!empty($luv['area'])){
if(empty($luv['area_module'])){
$luv['reg_area']=$this->convert_sign_match($luv['area']);
$luv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_area']);
$luv['reg_area']=str_replace('(*)', '[\s\S]*?', $luv['reg_area']);
}else{
$luv['reg_area']=$luv['area'];
}
$luv['reg_area_module']=$luv['area_module'];
}
if(empty($luv['url_rule_module'])){
if(!empty($luv['url_rule'])){
$luv['reg_url']=$this->convert_sign_match($luv['url_rule']);
$luv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_url']);
$luv['reg_url']=str_replace ( '(*)', '[\s\S]*?', $luv['reg_url'] );
}else{
$luv['reg_url']='\bhref=[\'\"](?P<match>[^\'\"\<\>]+?)[\'\"]';
}
$luv['url_merge']=$this->set_merge_default($luv['reg_url'], $luv['url_merge']);
}elseif('xpath'==$luv['url_rule_module']){
if(!empty($luv['url_rule'])){
$luv['reg_url']=$luv['url_rule'];
}else{
$luv['reg_url']='//a';
}
$luv['url_merge']=$this->set_merge_default('(?P<match>.+)', $luv['url_merge']);
}elseif('json'==$luv['url_rule_module']){
$luv['reg_url']=$luv['url_rule'];
$luv['url_merge']=$this->set_merge_default('(?P<match>.+)', $luv['url_merge']);
}
$luv['reg_url_module']=$luv['url_rule_module'];
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($luv['url_must'])){
$luv['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_must']);
$luv['url_must']=str_replace('(*)', '[\s\S]*?', $luv['url_must']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($luv['url_ban'])){
$luv['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_ban']);
$luv['url_ban']=str_replace('(*)', '[\s\S]*?', $luv['url_ban']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$config['level_urls'][$luk]=$luv;
$config['new_level_urls'][$luv['name']]=$luv;
}
}
$relation_urls=array();
if(!empty($config['relation_urls'])){
foreach ($config['relation_urls'] as $ruv){
2.2
2019-06-23 02:20:58 +00:00
if(!empty($ruv['area'])){
if(empty($ruv['area_module'])){
$ruv['reg_area']=$this->convert_sign_match($ruv['area']);
$ruv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_area']);
$ruv['reg_area']=str_replace('(*)', '[\s\S]*?', $ruv['reg_area']);
}else{
$ruv['reg_area']=$ruv['area'];
}
$ruv['reg_area_module']=$ruv['area_module'];
}
2.0
2019-02-19 09:52:13 +00:00
if(empty($ruv['url_rule_module'])){
$ruv['reg_url']=$this->convert_sign_match($ruv['url_rule']);
$ruv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_url']);
$ruv['reg_url']=str_replace('(*)', '[\s\S]*?', $ruv['reg_url']);
$ruv['url_merge']=$this->set_merge_default($ruv['reg_url'], $ruv['url_merge']);
}elseif(in_array($ruv['url_rule_module'],array('xpath','json'))){
$ruv['reg_url']=$ruv['url_rule'];
$ruv['url_merge']=$this->set_merge_default('(?P<match>.+)', $ruv['url_merge']);
}
$ruv['reg_url_module']=$ruv['url_rule_module'];
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$relation_urls[$ruv['name']]=$ruv;
}
}
$relation_depth_urls=array();
foreach ($relation_urls as $ruv){
$rDepth=0;
$rFuName=$ruv['page'];
if(empty($rFuName)){
$rDepth=0;
}else{
$passRelation=false;
$rFuPage=$rFuName;
do{
if(empty($relation_urls[$rFuPage])){
$passRelation=true;
break;
}
$rFuPage=$relation_urls[$rFuPage]['page'];
if($rFuPage==$rFuName){
$passRelation=true;
break;
}
$rDepth++;
}while(!empty($rFuPage));
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($passRelation){
continue;
}
}
$relation_depth_urls[$rDepth][$ruv['name']]=$ruv;
}
ksort($relation_depth_urls);
$config['new_relation_urls']=array();
foreach ($relation_depth_urls as $rurls){
if(is_array($rurls)){
$config['new_relation_urls']=array_merge($config['new_relation_urls'],$rurls);
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['field_list'])){
foreach ($config['field_list'] as $fk=>$fv){
if('rule'==$fv['module']){
$fv['reg_rule']=$this->convert_sign_match($fv['rule']);
$fv['reg_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $fv['reg_rule']);
$fv['reg_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_rule']);
$fv['rule_merge']=$this->set_merge_default($fv['reg_rule'], $fv['rule_merge']);
}elseif('extract'==$fv['module']){
if(!empty($fv['extract_rule'])){
$fv['reg_extract_rule']=$this->convert_sign_match($fv['extract_rule']);
$fv['reg_extract_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1",$fv['reg_extract_rule']);
$fv['reg_extract_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_extract_rule']);
$fv['extract_rule_merge']=$this->set_merge_default($fv['reg_extract_rule'], '');
}
}
$config['field_list'][$fk]=$fv;
}
}
if(!empty($config['field_process'])){
foreach ($config['field_process'] as $k=>$v){
$config['field_process'][$k]=$this->initProcess($v);
}
}
if(!empty($config['common_process'])){
$config['common_process']=$this->initProcess($config['common_process']);
}
if(!empty($config['paging']['area'])){
if(empty($config['paging']['area_module'])){
$config['paging']['reg_area']=$this->convert_sign_match($config['paging']['area']);
$config['paging']['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_area']);
$config['paging']['reg_area']=str_replace('(*)', '[\s\S]*?', $config['paging']['reg_area']);
$config['paging']['reg_area_merge']=$this->set_merge_default($config['paging']['reg_area'], '');
}else{
$config['paging']['reg_area']=$config['paging']['area'];
}
$config['paging']['reg_area_module']=$config['paging']['area_module'];
}
if(!empty($config['paging']['url_rule'])){
if(empty($config['paging']['url_rule_module'])){
$config['paging']['reg_url']=$this->convert_sign_match($config['paging']['url_rule']);
$config['paging']['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_url']);
$config['paging']['reg_url']=str_replace ( '(*)', '[\s\S]*?', $config['paging']['reg_url'] );
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$config['paging']['url_merge']=$this->set_merge_default($config['paging']['reg_url'], $config['paging']['url_merge']);
if(empty($config['paging']['url_merge'])){
$config['paging']['url_merge']=cp_sign('match');
}
}else{
$config['paging']['reg_url']=$config['paging']['url_rule'];
$config['paging']['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['paging']['url_merge']);
}
$config['paging']['reg_url_module']=$config['paging']['url_rule_module'];
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['paging']['url_must'])){
$config['paging']['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_must']);
$config['paging']['url_must']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_must']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($config['paging']['url_ban'])){
$config['paging']['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_ban']);
$config['paging']['url_ban']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_ban']);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$module_normal_fields=array();
$module_extract_fields=array();
$module_merge_fields=array();
if(!empty($config['field_list'])){
foreach ($config['field_list'] as $fk=>$fv){
$fieldModule=strtolower($fv['module']);
$fieldConfig=array('field'=>$fv,'process'=>$config['field_process'][$fk]);
if('extract'==$fieldModule){
$module_extract_fields[$fv['name']]=$fieldConfig;
}elseif('merge'==$fieldModule){
$module_merge_fields[$fv['name']]=$fieldConfig;
}else{
$module_normal_fields[$fv['name']]=$fieldConfig;
}
}
}
$config['new_field_list']=array_merge($module_normal_fields,$module_extract_fields,$module_merge_fields);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$new_paging_fields=array(
2.2
2019-06-23 02:20:58 +00:00
'normal'=>array(),
'extract'=>array(),
'merge'=>array(),
2.0
2019-02-19 09:52:13 +00:00
);
if(!empty($config['paging_fields'])){
2.2
2019-06-23 02:20:58 +00:00
$pagingFields=array();
foreach ($config['paging_fields'] as $pfield){
$pagingFields[$pfield['field']]=$pfield;
}
if(!empty($pagingFields['::all'])){
$fieldAllParams=$pagingFields['::all'];
unset($pagingFields['::all']);
foreach ($config['new_field_list'] as $k=>$v){
if(empty($pagingFields[$k])){
$fieldAllParams['field']=$k;
$pagingFields[$k]=$fieldAllParams;
}
}
}
$config['paging_fields']=$pagingFields;
unset($pagingFields);
2.0
2019-02-19 09:52:13 +00:00
foreach ($config['paging_fields'] as $pfk=>$pfield){
$pfield['delimiter']=str_replace(array('\r','\n'), array("\r","\n"), $pfield['delimiter']);
$config['paging_fields'][$pfk]=$pfield;
if(!empty($module_normal_fields[$pfield['field']])){
$new_paging_fields['normal'][$pfield['field']]=$pfield;
}elseif(!empty($module_extract_fields[$pfield['field']])){
$new_paging_fields['extract'][$pfield['field']]=$pfield;
}elseif(!empty($module_merge_fields[$pfield['field']])){
$new_paging_fields['merge'][$pfield['field']]=$pfield;
}
}
}
$config['new_paging_fields']=array_merge($new_paging_fields['normal'],$new_paging_fields['extract'],$new_paging_fields['merge']);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$config=array_merge($config,$newConfig);
return $config;
}
2.2
2019-06-23 02:20:58 +00:00
/*采集级别网址*/
public function collLevelUrls($source_url,$curLevel=1){
$curLevel=$curLevel>0?$curLevel:0;
if($curLevel>0){
$nextLevel=0;
if(!empty($this->config['level_urls'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
if(!empty($this->config['level_urls'][$curLevel-1])){
if(!empty($this->config['level_urls'][$curLevel])){
$nextLevel=$curLevel+1;
2.0
2019-02-19 09:52:13 +00:00
}
}
}
2.2
2019-06-23 02:20:58 +00:00
$cont_urls=$this->getLevelUrls($source_url,$curLevel);
2.0
2019-02-19 09:52:13 +00:00
}else{
2.2
2019-06-23 02:20:58 +00:00
$cont_urls=$this->getContUrls($source_url);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
return array('urls'=>$cont_urls,'levelName'=>$this->config['level_urls'][$curLevel-1]['name'],'nextLevel'=>$nextLevel);
2.0
2019-02-19 09:52:13 +00:00
}
/*获取内容网址*/
public function getContUrls($source_url){
if(empty($source_url)){
return $this->error('请输入起始网址');
}
$config=array(
2.2
2019-06-23 02:20:58 +00:00
'reg_area'=>$this->config['reg_source_cont'],
'reg_area_module'=>$this->config['area_module'],
'reg_url'=>$this->config['reg_source_cont_url'],
'reg_url_module'=>$this->config['url_rule_module'],
'url_merge'=>$this->config['url_merge'],
'url_must'=>$this->config['url_must'],
'url_ban'=>$this->config['url_ban'],
2.0
2019-02-19 09:52:13 +00:00
);
return $this->_get_urls($source_url, $config);
}
/*获取多级网址*/
public function getLevelUrls($parent_url,$level=1){
$level=$level>1?$level:1;
$config=$this->config['level_urls'][$level-1];
if(empty($config)){
return $this->error('没有'.($level).'级网址规则');
}
if(empty($config['reg_url'])){
return $this->error('必须填写多级“提取网址规则”');
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(empty($parent_url)){
return $this->error('请输入父级网址');
}
return $this->_get_urls($parent_url, $config,true);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
/*获取分页链接*/
public function getPagingUrls($from_url,$html,$is_test=false){
$paging_urls=array();
if($this->config['paging']['open']){
if(empty($html)){
$html=$this->get_html($from_url);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($this->config['paging']['reg_url'])){
if(!empty($this->config['new_paging_fields'])){
$base_url=$this->match_base_url($from_url, $html);
$domain_url=$this->match_domain_url($from_url);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$paging_area='';
if(!empty($this->config['paging']['reg_area'])){
if(empty($this->config['paging']['reg_area_module'])){
$sign_match=$this->sign_addslashes(cp_sign('match','(?P<num>\d*)'));
if(preg_match_all('/'.$sign_match.'/i', $this->config['paging']['reg_area_merge'],$match_signs)){
if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){
$re_match=array();
foreach($match_signs['num'] as $ms_k=>$ms_v){
$re_match[$ms_k]=$m_paging_area['match'.$ms_v];
}
$paging_area=str_replace($match_signs[0], $re_match, $this->config['paging']['reg_area_merge']);
}
}else{
if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){
$paging_area=$m_paging_area[0];
}
}
}elseif('json'==$this->config['paging']['reg_area_module']){
2.2
2019-06-23 02:20:58 +00:00
$paging_area=$this->rule_module_json_data(array('json'=>$this->config['paging']['reg_area'],'json_arr'=>'jsonencode'),$html);
2.0
2019-02-19 09:52:13 +00:00
}elseif('xpath'==$this->config['paging']['reg_area_module']){
$paging_area=$this->rule_module_xpath_data(array('xpath'=>$this->config['paging']['reg_area'],'xpath_attr'=>'outerHtml'),$html);
}
}else{
$paging_area=$html;
}
if(!empty($paging_area)){
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($this->config['paging']['url_complete'])){
$paging_area=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche_p_a) use ($base_url,$domain_url){
return \skycaiji\admin\event\Cpattern::create_complete_url($matche_p_a[1], $base_url, $domain_url);
},$paging_area);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$m_paging_urls=$this->rule_match_urls($this->config['paging'],$paging_area,true);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
foreach ($m_paging_urls as $purl){
if(!empty($this->config['paging']['url_must'])){
if(!preg_match('/'.$this->config['paging']['url_must'].'/i', $purl)){
continue;
}
}
if(!empty($this->config['paging']['url_ban'])){
if(preg_match('/'.$this->config['paging']['url_ban'].'/i', $purl)){
continue;
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($from_url==$purl){
continue;
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(strpos($purl,' ')==false){
$paging_urls[]=$purl;
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($paging_urls)){
$paging_urls=array_filter($paging_urls);
$paging_urls=array_unique($paging_urls);
$paging_urls=array_values($paging_urls);
}else{
if($is_test){
return $this->error('未获取到分页链接，请检查分页链接规则');
}
}
}else{
if($is_test){
return $this->error('未获取到分页区域，请检查分页区域规则');
}
}
}else{
if($is_test){
return $this->error('请添加分页内容字段');
}
}
}else{
if($is_test){
return $this->error('必须填写分页链接规则');
}
}
}else{
if($is_test){
return $this->error('未开启分页');
}
}
return $paging_urls;
}
/*设置字段值*/
2.2
2019-06-23 02:20:58 +00:00
public function setField($field_config,$cur_url,$html,$cont_url){
$cur_url_md5=md5($cur_url);
2.0
2019-02-19 09:52:13 +00:00
$field_process=$field_config['process'];
$field_params=$field_config['field'];
$module=strtolower($field_params['module']);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($field_params['source'])&&in_array($module, array('rule','xpath','json','auto'))){
$field_source_url='';
$source_echo_msg='——采集';
if('source_url'==$field_params['source']){
$field_source_url=$this->cur_source_url;
$source_echo_msg.='起始页';
}elseif(preg_match('/^relation_url:(.+)$/i', $field_params['source'],$relationName)){
$relationName=$relationName[1];
2.2
2019-06-23 02:20:58 +00:00
$field_source_url=$this->getRelationUrl($relationName, $cur_url, $html);
2.0
2019-02-19 09:52:13 +00:00
$source_echo_msg.="关联页“{$relationName}”";
}elseif(preg_match('/^level_url:(.+)$/i', $field_params['source'],$levelName)){
$levelName=$levelName[1];
if(empty($this->config['new_level_urls'][$levelName])){
return;
}
if(empty($this->cur_level_urls[$levelName])){
return;
}
$field_source_url=$this->cur_level_urls[$levelName];
$source_echo_msg.="多级页“{$levelName}”";
}
if(empty($field_source_url)){
return;
}
2.2
2019-06-23 02:20:58 +00:00
if($field_source_url!=$cur_url){
$cur_url=$field_source_url;
2.0
2019-02-19 09:52:13 +00:00
$this->echo_msg($source_echo_msg."：<a href='{$field_source_url}' target='_blank'>{$field_source_url}</a>",'black');
$html=$this->get_html($field_source_url,true);
}
}
static $fieldArr1=array('words','num','time','list');
static $fieldArr2=array('auto','json');
static $baseUrls=array();
static $domainUrls=array();
2.2
2019-06-23 02:20:58 +00:00
$urlMd5=md5($cur_url);
2.0
2019-02-19 09:52:13 +00:00
if(empty($baseUrls[$urlMd5])){
2.2
2019-06-23 02:20:58 +00:00
$baseUrls[$urlMd5]=$this->match_base_url($cur_url, $html);
2.0
2019-02-19 09:52:13 +00:00
}
if(empty($domainUrls[$urlMd5])){
2.2
2019-06-23 02:20:58 +00:00
$domainUrls[$urlMd5]=$this->match_domain_url($cur_url);
2.0
2019-02-19 09:52:13 +00:00
}
$base_url=$baseUrls[$urlMd5];
$domain_url=$domainUrls[$urlMd5];
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$val='';
$field_func='field_module_'.$module;
if(method_exists($this, $field_func)){
if('extract'==$module){
2.2
2019-06-23 02:20:58 +00:00
if(is_array($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5])){
2.0
2019-02-19 09:52:13 +00:00
$val=array();
2.2
2019-06-23 02:20:58 +00:00
foreach ($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5] as $k=>$v){
2.0
2019-02-19 09:52:13 +00:00
$extract_field_val=array(
'value'=>$v,
2.2
2019-06-23 02:20:58 +00:00
'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5][$k],
2.0
2019-02-19 09:52:13 +00:00
);
$val[$k]=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url);
}
}else{
$extract_field_val=array(
2.2
2019-06-23 02:20:58 +00:00
'value'=>$this->field_val_list[$field_params['extract']]['values'][$cur_url_md5],
'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5],
2.0
2019-02-19 09:52:13 +00:00
);
$val=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url);
}
}elseif('merge'==$module){
if(empty($this->first_loop_field)){
$cur_field_val_list=array();
foreach ($this->field_val_list as $k=>$v){
$cur_field_val_list[$k]=array(
2.2
2019-06-23 02:20:58 +00:00
'value'=>$v['values'][$cur_url_md5],
'img'=>$v['imgs'][$cur_url_md5]
2.0
2019-02-19 09:52:13 +00:00
);
}
$val=$this->field_module_merge($field_params,$cur_field_val_list);
}else{
$val=array();
2.2
2019-06-23 02:20:58 +00:00
foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){
2.0
2019-02-19 09:52:13 +00:00
$cur_field_val_list=array();
foreach ($this->field_val_list as $k=>$v){
$cur_field_val_list[$k]=array(
2.2
2019-06-23 02:20:58 +00:00
'value'=>(is_array($v['values'][$cur_url_md5])?$v['values'][$cur_url_md5][$v_k]:$v['values'][$cur_url_md5]),
'img'=>(is_array($v['imgs'][$cur_url_md5][$v_k])?$v['imgs'][$cur_url_md5][$v_k]:$v['imgs'][$cur_url_md5])
2.0
2019-02-19 09:52:13 +00:00
);
}
$val[$v_k]=$this->field_module_merge($field_params,$cur_field_val_list);
}
}
}elseif(in_array($module,$fieldArr1)){
2.2
2019-06-23 02:20:58 +00:00
if($module=='words'){
$val=$this->$field_func($field_params);
}else{
if(empty($this->first_loop_field)){
$val=$this->$field_func($field_params);
}else{
$val=array();
foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){
$val[$v_k]=$this->$field_func($field_params);
}
}
}
2.0
2019-02-19 09:52:13 +00:00
}elseif(in_array($module,$fieldArr2)){
2.2
2019-06-23 02:20:58 +00:00
$val=$this->$field_func($field_params,$html,$cur_url);
2.0
2019-02-19 09:52:13 +00:00
}else{
$val=$this->$field_func($field_params,$html);
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$vals=null;
if(is_array($val)){
$is_loop=true;
$vals=array_values($val);
}else{
$is_loop=false;
$vals=array($val);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$field_name=$field_params['name'];
if(!isset($this->field_val_list[$field_name])){
$this->field_val_list[$field_name]=array('values'=>array(),'imgs'=>array());
}
2.2
2019-06-23 02:20:58 +00:00
$cont_url_md5=empty($cont_url)?$cur_url_md5:md5($cont_url);
2.0
2019-02-19 09:52:13 +00:00
foreach ($vals as $v_k=>$val){
2.2
2019-06-23 02:20:58 +00:00
$loopIndex=$is_loop?$v_k:-1;
2.0
2019-02-19 09:52:13 +00:00
if(!empty($field_process)){
2.2
2019-06-23 02:20:58 +00:00
$val=$this->process_field($val,$field_process,$cur_url_md5,$loopIndex,$cont_url_md5);
2.0
2019-02-19 09:52:13 +00:00
}
if(!empty($this->config['common_process'])){
2.2
2019-06-23 02:20:58 +00:00
$val=$this->process_field($val,$this->config['common_process'],$cur_url_md5,$loopIndex,$cont_url_md5);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5])){
if(empty($this->first_loop_field)){
foreach ($this->field_val_list as $f_k=>$f_v){
unset($this->field_val_list[$f_k]['values'][$cur_url_md5]);
unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]);
}
return;
}else{
if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5][$loopIndex])){
if(!$is_loop){
foreach ($this->field_val_list as $f_k=>$f_v){
unset($this->field_val_list[$f_k]['values'][$cur_url_md5]);
unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]);
}
return;
}else{
foreach ($this->field_val_list as $f_k=>$f_v){
if(is_array($this->field_val_list[$f_k]['values'][$cur_url_md5])){
unset($this->field_val_list[$f_k]['values'][$cur_url_md5][$v_k]);
}
if(is_array($this->field_val_list[$f_k]['imgs'][$cur_url_md5])){
unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5][$v_k]);
}
}
continue;
}
}
}
}
2.0
2019-02-19 09:52:13 +00:00
$val=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){
return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url);
},$val);
$val=preg_replace_callback('/(?<=\bsrc\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){
return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url);
},$val);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($is_loop){
2.2
2019-06-23 02:20:58 +00:00
if(!isset($this->field_val_list[$field_name]['values'][$cur_url_md5])){
$this->field_val_list[$field_name]['values'][$cur_url_md5]=array();
$this->field_val_list[$field_name]['imgs'][$cur_url_md5]=array();
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$val;
2.0
2019-02-19 09:52:13 +00:00
}else{
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['values'][$cur_url_md5]=$val;
2.0
2019-02-19 09:52:13 +00:00
}
if(!empty($GLOBALS['config']['caiji']['download_img'])&&!empty($val)){
$valImgs=array();
if(preg_match_all('/<img[^<>]*\bsrc=[\'\"]*(\w+\:\/\/[^\'\"\s]+)[\'\"]*/i',$val,$imgUrls)){
$valImgs=is_array($imgUrls[1])?$imgUrls[1]:array();
}
if('extract'==$module&&'cover'==$field_params['extract_module']){
$valImgs=array_merge($valImgs,array($val));
}
$noImgVal=preg_replace_callback('/\{\[img\]\}(http[s]{0,1}\:\/\/[^\s]+?)\{\[\/img\]\}/i',function($matche) use (&$valImgs){
$valImgs[]=$matche[1];
return $matche[1];
},$val);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($noImgVal!=$val){
if($is_loop){
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$noImgVal;
2.0
2019-02-19 09:52:13 +00:00
}else{
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['values'][$cur_url_md5]=$noImgVal;
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!empty($valImgs)){
$valImgs=array_unique($valImgs);
$valImgs=array_values($valImgs);
if($is_loop){
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['imgs'][$cur_url_md5][$v_k]=$valImgs;
2.0
2019-02-19 09:52:13 +00:00
}else{
2.2
2019-06-23 02:20:58 +00:00
$this->field_val_list[$field_name]['imgs'][$cur_url_md5]=$valImgs;
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
}
2.0
2019-02-19 09:52:13 +00:00
}
}
/*设置分页的字段列表值*/
public function setPagingFields($cont_url,$page_url){
$contMd5=md5($cont_url);
$pageMd5=md5($page_url);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(empty($page_url)){
return $this->error('请输入分页网址');
}
if(!preg_match('/^\w+\:\/\//',$page_url)){
return $this->error($page_url.'网址不完整');
}
if(empty($this->config['paging']['max'])||(count($this->used_paging_urls[$contMd5])<$this->config['paging']['max'])){
$this->set_html_interval();
$this->echo_msg("——采集分页：<a href='{$page_url}' target='_blank'>{$page_url}</a>",'black');
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$html=$this->get_html($page_url);
if(empty($html)){
return $this->error('分页获取失败：'.$page_url);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(!isset($this->used_paging_urls[$contMd5][$pageMd5])){
$this->used_paging_urls[$contMd5][$pageMd5]=$page_url;
foreach ($this->config['new_paging_fields'] as $v){
2.2
2019-06-23 02:20:58 +00:00
$this->setField($this->config['new_field_list'][$v['field']],$page_url,$html,$cont_url);
2.0
2019-02-19 09:52:13 +00:00
}
}
$paging_urls=$this->getPagingUrls($page_url,$html);
if(!empty($paging_urls)){
$nextUrl='';
foreach ($paging_urls as $purl){
if(!isset($this->used_paging_urls[$contMd5][md5($purl)])&&$cont_url!=$purl){
$nextUrl=$purl;
break;
}
}
if(!empty($nextUrl)){
$this->setPagingFields($cont_url,$nextUrl);
}
}
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
/**
* 获取关联页网址
* @param unknown $name 关联页名称
* @param unknown $cont_url 内容页网址
* @param unknown $html 内容页源码
* @return string
*/
public function getRelationUrl($name,$cont_url,$html){
if(empty($html)){
$html=$this->get_html($cont_url,true);
}
if(empty($html)){
return '';
}
$relation_url=$this->config['new_relation_urls'][$name];
if(empty($relation_url)){
return '';
}
2.2
2019-06-23 02:20:58 +00:00
2.1
2019-03-14 09:13:21 +00:00
if(empty($relation_url['page'])){
if(!isset($this->relation_url_list[$cont_url][$name])){
2.2
2019-06-23 02:20:58 +00:00
$html=$this->rule_match_area($relation_url, $html);
if(empty($html)){
return '';
}
2.1
2019-03-14 09:13:21 +00:00
$relationUrl=$this->rule_match_urls($relation_url, $html);
$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
$this->relation_url_list[$cont_url][$name]=$relationUrl;
}else{
$relationUrl=$this->relation_url_list[$cont_url][$name];
}
}else{
2.0
2019-02-19 09:52:13 +00:00
2.1
2019-03-14 09:13:21 +00:00
$page=$relation_url['page'];
$pass=false;
$depth_pages=array();
$depth=0;
while(!empty($page)){
2.0
2019-02-19 09:52:13 +00:00
2.1
2019-03-14 09:13:21 +00:00
if($page==$name){
$pass=true;
break;
}
if(empty($this->config['new_relation_urls'][$page])){
$pass=true;
break;
}
$depth++;
$depth_pages[$depth]=$page;
$page=$this->config['new_relation_urls'][$page]['page'];
2.0
2019-02-19 09:52:13 +00:00
}
2.1
2019-03-14 09:13:21 +00:00
if($pass){
2.0
2019-02-19 09:52:13 +00:00
2.1
2019-03-14 09:13:21 +00:00
return '';
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
krsort($depth_pages);
2.1
2019-03-14 09:13:21 +00:00
$contPage=reset($depth_pages);
$relationUrl='';
if(isset($contPage)){
if(!isset($this->relation_url_list[$cont_url][$contPage])){
2.2
2019-06-23 02:20:58 +00:00
$html=$this->rule_match_area($this->config['new_relation_urls'][$contPage], $html);
if(empty($html)){
return '';
}
2.1
2019-03-14 09:13:21 +00:00
$relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$contPage], $html);
$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
$this->relation_url_list[$cont_url][$contPage]=$relationUrl;
}else{
$relationUrl=$this->relation_url_list[$cont_url][$contPage];
}
}
$depth_pages=array_slice($depth_pages, 1);
$depth_pages=is_array($depth_pages)?$depth_pages:array();
$depth_pages[]=$relation_url['name'];
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
foreach ($depth_pages as $page){
if(empty($relationUrl)){
return '';
}
if(!isset($this->relation_url_list[$cont_url][$page])){
$relationHtml=$this->get_html($relationUrl,true);
2.2
2019-06-23 02:20:58 +00:00
$relationHtml=$this->rule_match_area($this->config['new_relation_urls'][$page], $relationHtml);
2.0
2019-02-19 09:52:13 +00:00
if(empty($relationHtml)){
return '';
}
2.1
2019-03-14 09:13:21 +00:00
$relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$page],$relationHtml);
$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$this->relation_url_list[$cont_url][$page]=$relationUrl;
}else{
$relationUrl=$this->relation_url_list[$cont_url][$page];
}
}
}
2.2
2019-06-23 02:20:58 +00:00
2.1
2019-03-14 09:13:21 +00:00
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
return $relationUrl;
}
/*获取内容页字段列表，这里是入口*/
public function getFields($cont_url){
$this->field_val_list=array();
$this->first_loop_field=null;
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if(empty($cont_url)){
return $this->error('请输入内容页网址');
}
if(!preg_match('/^\w+\:\/\//',$cont_url)){
return $this->error($cont_url.'网址不完整');
}
$html=$this->get_html($cont_url,false,$this->config['url_post']);
if(empty($html)){
return $this->error('抓取页面失败');
}
foreach($this->config['new_field_list'] as $field_config){
2.2
2019-06-23 02:20:58 +00:00
$this->setField($field_config,$cont_url,$html,$cont_url);
2.0
2019-02-19 09:52:13 +00:00
}
$paging_urls=$this->getPagingUrls($cont_url,$html);
if(!empty($paging_urls)){
$this->setPagingFields($cont_url,reset($paging_urls));
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$val_list=array();
if(!empty($this->field_val_list)){
if(empty($this->first_loop_field)){
foreach ($this->field_val_list as $fieldName=>$fieldVal){
$val_values=array_filter($fieldVal['values']);
$val_values=implode($this->config['new_paging_fields'][$fieldName]['delimiter'], $val_values);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$val_imgs=array();
if(!empty($fieldVal['imgs'])){
foreach ($fieldVal['imgs'] as $v){
if(!empty($v)){
if(is_array($v)){
$val_imgs=array_merge($val_imgs,$v);
}else{
$val_imgs[]=$v;
}
}
}
if(!empty($val_imgs)){
$val_imgs=array_unique($val_imgs);
$val_imgs=array_filter($val_imgs);
$val_imgs=array_values($val_imgs);
}
}
$val_list[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs);
}
}else{
foreach ($this->field_val_list[$this->first_loop_field]['values'] as $page_key=>$page_vals){
if(empty($page_vals)){
continue;
}
foreach ($page_vals as $loop_index=>$loop_val){
$vals=array();
foreach ($this->field_val_list as $fieldName=>$fieldVals){
if(is_array($fieldVals['values'][$page_key])){
$val_values=$fieldVals['values'][$page_key][$loop_index];
$val_imgs=$fieldVals['imgs'][$page_key][$loop_index];
}else{
$val_values=$fieldVals['values'][$page_key];
$val_imgs=$fieldVals['imgs'][$page_key];
}
if(!empty($val_imgs)){
$val_imgs=array_unique($val_imgs);
$val_imgs=array_filter($val_imgs);
$val_imgs=array_values($val_imgs);
}
$vals[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs);
}
$val_list[]=$vals;
}
}
}
}
return $val_list?$val_list:array();
}
2.2
2019-06-23 02:20:58 +00:00
/*设置数据处理，保存config时使用*/
public function setProcess($processList){
if(is_array($processList)){
$processList=array_array_map('trim',$processList);
foreach ($processList as $k=>$v){
$v['module']=strtolower($v['module']);
if(!empty($v['title'])){
$v['title']=str_replace(array("'",'"'),'',strip_tags($v['title']));
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
if('html'==$v['module']){
$v['html_allow']=$this->clear_tags($v['html_allow']);
$v['html_allow']=implode(',', $v['html_allow']);
$v['html_filter']=$this->clear_tags($v['html_filter']);
$v['html_filter']=implode(',', $v['html_filter']);
}elseif('filter'==$v['module']){
if(preg_match_all('/[^\r\n]+/', $v['filter_list'],$filterList)){
$filterList=array_filter(array_unique($filterList[0]));
$v['filter_list']=implode("\r\n",$filterList);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$v['filter_list']=trim($v['filter_list']);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$processList[$k]=$v;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$processList=array_values($processList);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
return $processList;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
/*初始化数据处理，初始化config时使用*/
public function initProcess($processList){
if(!empty($processList)){
foreach ($processList as $k=>$v){
if('replace'==$v['module']){
$v['replace_from']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $v['replace_from']);
$v['replace_from']=str_replace('(*)', '[\s\S]*?', $v['replace_from']);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$processList[$k]=$v;
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
return $processList;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
/*统一：获取网址列表*/
public function _get_urls($source_url,$config,$is_level=false){
$is_level=$is_level?'多级':'';
$html=$this->get_html($source_url);
if(empty($html)){
return $this->error($is_level.'页面为空');
}
$base_url=$this->match_base_url($source_url, $html);
$domain_url=$this->match_domain_url($source_url);
$html=$this->rule_match_area($config, $html);
if(empty($html)){
return $this->error("未提取到{$is_level}区域内容！");
}
$cont_urls=$this->rule_match_urls($config, $html);
$cont_urls1=array();
if(isset($this->config['url_op'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$op_not_complete=in_array('not_complete',$this->config['url_op'])?true:false;
}else{
if(isset($this->config['url_complete'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$op_not_complete=$this->config['url_complete']?false:true;
}else{
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$op_not_complete=false;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
}
foreach ($cont_urls as $cont_url){
if(!$op_not_complete){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$cont_url=$this->create_complete_url($cont_url, $base_url, $domain_url);
}
if(!empty($config['url_must'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
if(!preg_match('/'.$config['url_must'].'/i', $cont_url)){
continue;
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
if(!empty($config['url_ban'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
if(preg_match('/'.$config['url_ban'].'/i', $cont_url)){
continue;
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
if(!empty($cont_url)){
if(strpos($cont_url,' ')==false){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$cont_urls1[]=$cont_url;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
}
}
$cont_urls=$cont_urls1;
unset($cont_urls1);
if(empty($cont_urls)){
return $this->error("未获取到".($is_level?$is_level:'内容')."网址！");
}else{
if(!empty($this->config['url_reverse'])){
$cont_urls=array_reverse($cont_urls);
}
if(!empty($this->config['url_post'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$postParams=array();
if(!empty($this->config['url_posts']['names'])){
foreach ($this->config['url_posts']['names'] as $k=>$v){
if (!empty($v)){
$postParams[]=$v.'='.rawurlencode($this->config['url_posts']['vals'][$k]);
}
}
}
if(!empty($postParams)){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$postParams=implode('&', $postParams);
foreach ($cont_urls as $k=>$v){
$v.=strpos($v,'?')===false?'?':'&';
$v.=$postParams;
$cont_urls[$k]=$v;
}
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
return array_values($cont_urls);
2.0
2019-02-19 09:52:13 +00:00
}
}
2.2
2019-06-23 02:20:58 +00:00
/*执行采集返回未使用的网址*/
public function _collect_unused_cont_urls($cont_urls=array(),$echo_str=''){
$mcollected=model('Collected');
$count_conts=count($cont_urls);
if($this->config['url_repeat']){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$db_cont_urls=array();
2.0
2019-02-19 09:52:13 +00:00
}else{
2.2
2019-06-23 02:20:58 +00:00
$db_cont_urls=$mcollected->getUrlByUrl($cont_urls);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$unused_cont_urls=array();
$count_used=0;
if(!empty($cont_urls)){
foreach ($cont_urls as $cont_url){
if(array_key_exists(md5($cont_url), $this->used_cont_urls)){
$count_used++;
}elseif(in_array($cont_url, $db_cont_urls)){
$count_used++;
}else{
$unused_cont_urls[md5($cont_url)]=$cont_url;
2.0
2019-02-19 09:52:13 +00:00
}
}
}
2.2
2019-06-23 02:20:58 +00:00
if($count_used>0){
$count_used=min(count($cont_urls),$count_used);
$this->echo_msg($echo_str.'采集到'.$count_conts.'条网址，<span style="color:orange">'.$count_used.'</span>条重复，<span style="color:green">'.(count($unused_cont_urls)).'</span>条有效','black');
}else{
$this->echo_msg($echo_str.'采集到<span style="color:green">'.$count_conts.'</span>条有效网址','black');
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
return $unused_cont_urls;
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
/*执行级别采集*/
public function _collect_level($source_url,$level=1){
$end_echo='</div>';
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$level=max(1,$level);
$level_str='';
for($i=1;$i<$level;$i++){
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
$next_level_str=$level_str;
if($level<=1){
$this->cur_level_urls=array();
}
$this->echo_msg('','',true,'<div style="padding-left:30px;">');
$level_data=$this->collLevelUrls($source_url,$level);
$this->echo_msg($level_str.'抓取到'.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址'.count($level_data['urls']).'条','black');
$mcollected=model('Collected');
$mcacheLevel=CacheModel::getInstance('level_url');
if(!empty($level_data['urls'])){
$level_urls=array();
foreach ($level_data['urls'] as $level_url){
$level_urls["level_{$level}:{$level_url}"]=$level_url;
}
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
$level_interval=$GLOBALS['config']['caiji']['interval']*60;
$time_interval_list=array();
$cacheLevels=$mcacheLevel->db()->where(array('cname'=>array('in',array_map('md5', array_keys($level_urls)))))->column('dateline','cname');
if(!empty($cacheLevels)){
$count_db_used=0;
$sortLevels=array('undb'=>array(),'db'=>array());
2.0
2019-02-19 09:52:13 +00:00
foreach ($level_urls as $level_key=>$level_url){
$md5_level_key=md5($level_key);
if(!isset($cacheLevels[$md5_level_key])){
$sortLevels['undb'][$level_key]=$level_url;
}else{
$time_interval=abs(NOW_TIME-$cacheLevels[$md5_level_key]);
if($time_interval<$level_interval){
$this->used_level_urls[$level_key]=1;
$count_db_used++;
$time_interval_list[]=$time_interval;
}else{
$sortLevels['db'][$level_key]=$level_url;
}
}
}
if($count_db_used>0){
$this->echo_msg($level_str.$count_db_used.'条已采集网址被过滤，下次采集需等待'.($level_interval-max($time_interval_list)).'秒，<a href="'
2.2
2019-06-23 02:20:58 +00:00
.url('Admin/Setting/caiji').'" target="_blank">设置间隔</a>','black');
2.0
2019-02-19 09:52:13 +00:00
if(count($level_urls)<=$count_db_used){
$this->echo_msg($level_str.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址采集完毕！','green',true,$end_echo);
return 'completed';
}
}
$level_urls=array_merge($sortLevels['undb'],$sortLevels['db']);
unset($sortLevels);
unset($cacheLevels);
}
$level_data['urls']=$level_urls;
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$finished_source=true;
$cur_level_i=0;;
if(!empty($level_data['urls'])){
foreach ($level_data['urls'] as $level_key=>$level_url){
$cur_level_i++;
if(array_key_exists($level_key,$this->used_level_urls)){
continue;
}
$this->cur_level_urls[$this->config['level_urls'][$level-1]['name']]=$level_url;
$this->echo_msg("{$next_level_str}分析第{$level}级：<a href='{$level_url}' target='_blank'>{$level_url}</a>",'black');
if($level_data['nextLevel']>0){
$return_msg=$this->_collect_level($level_url,$level_data['nextLevel']);
if($return_msg=='completed'){
$this->echo_msg('','',true,$end_echo);
return $return_msg;
}
}else{
$cont_urls=$this->getContUrls($level_url);
$cont_urls=$this->_collect_unused_cont_urls($cont_urls,$next_level_str);
$this->cont_urls_list[$level_key]=$cont_urls;
$this->_collect_fields($next_level_str);
}
if($this->collect_num>0){
if(count($this->collected_field_list)>=$this->collect_num){
if($cur_level_i<count($level_data['urls'])){
$finished_source=false;
}
break;
}
}
}
}
if($finished_source){
$source_key='level_'.($level-1).':'.$source_url;
$this->used_level_urls[$source_key]=1;
$mcacheLevel->setCache(md5($source_key),$source_key);
if($level<=1){
$mcacheSource=CacheModel::getInstance('source_url');
$this->used_source_urls[$source_url]=1;
$mcacheSource->setCache(md5($source_url),$source_url);
}
}
$this->echo_msg('','',true,$end_echo);
}
/*采集字段列表*/
public function _collect_fields($echo_str=''){
$mcollected=model('Collected');
$mcacheSource=CacheModel::getInstance('source_url');
$mcacheLevel=CacheModel::getInstance('level_url');
$mcacheCont=CacheModel::getInstance('cont_url');
foreach ($this->cont_urls_list as $cont_key=>$cont_urls){
$source_type=0;
if('_source_is_url_'==$cont_key){
$source_type=0;
}elseif(strpos($cont_key,'level_')===0){
$source_type=2;
}else{
$source_type=1;
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($source_type==2){
if(array_key_exists($cont_key,$this->used_level_urls)){
continue;
}
}else{
if(array_key_exists($cont_key,$this->used_source_urls)){
continue;
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$finished_cont=true;
$cur_c_i=0;
foreach ($cont_urls as $cont_url){
$cur_c_i+=1;
$md5_cont_url=md5($cont_url);
if(array_key_exists($md5_cont_url,$this->used_cont_urls)){
continue;
}
if($this->config['url_repeat']||$mcollected->getCountByUrl($cont_url)<=0){
if(!empty($this->collected_field_list)){
if($this->set_html_interval()===true){
if(!$this->config['url_repeat']&&$mcollected->getCountByUrl($cont_url)>0){
$this->used_cont_urls[$md5_cont_url]=1;
continue;
}
}
}
2.2
2019-06-23 02:20:58 +00:00
2.1
2019-03-14 09:13:21 +00:00
if(input('?backstage')){
$backstageDate=CacheModel::getInstance('backstage_task')->db()->field('dateline')->where('cname',$this->collector['task_id'])->find();
if(empty($backstageDate)||$GLOBALS['backstage_task_runtime']<$backstageDate['dateline']){
unset($GLOBALS['backstage_task_ids'][$this->collector['task_id']]);
exit('终止进程');
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($mcacheCont->getCount($md5_cont_url)>0){
$this->used_cont_urls[$md5_cont_url]=1;
continue;
}
$mcacheCont->setCache($md5_cont_url, 1);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$this->echo_msg($echo_str."采集内容页：<a href='{$cont_url}' target='_blank'>{$cont_url}</a>",'black');
$field_vals_list=$this->getFields($cont_url);
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
$is_loop=empty($this->first_loop_field)?false:true;
if(!empty($field_vals_list)){
$is_real_time=false;
if(!empty($GLOBALS['config']['caiji']['real_time'])&&!empty($GLOBALS['real_time_release'])){
$is_real_time=true;
}
if(!$is_loop){
$field_vals_list=array($field_vals_list);
}else{
$loop_cont_urls=array();
foreach ($field_vals_list as $k=>$field_vals){
$loop_cont_urls[$k]=$cont_url.'#'.md5(serialize($field_vals));
}
if(!empty($loop_cont_urls)){
$loop_exists_urls=$mcollected->getUrlByUrl($loop_cont_urls);
if(!empty($loop_exists_urls)){
$loop_exists_urls=array_flip($loop_exists_urls);
foreach ($loop_cont_urls as $k=>$loop_cont_url){
if(isset($loop_exists_urls[$loop_cont_url])){
unset($field_vals_list[$k]);
}
}
$this->echo_msg($echo_str.'已过滤'.count($loop_exists_urls).'条重复数据','black');
}
}
2.2
2019-06-23 02:20:58 +00:00
if(isset($this->exclude_cont_urls[$md5_cont_url])){
$excludeNum=0;
foreach($this->exclude_cont_urls[$md5_cont_url] as $k=>$v){
$excludeNum+=count($v);
}
$this->echo_msg($echo_str.'通过数据处理排除了'.$excludeNum.'条数据','black');
}
$field_vals_list=array_values($field_vals_list);
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
foreach ($field_vals_list as $field_vals){
2.2
2019-06-23 02:20:58 +00:00
$collected_error='';
2.0
2019-02-19 09:52:13 +00:00
$collected_data=array('url'=>$cont_url,'fields'=>$field_vals);
if($is_loop){
$collected_data['url'].='#'.md5(serialize($field_vals));
2.2
2019-06-23 02:20:58 +00:00
}else{
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
if(isset($this->exclude_cont_urls[$md5_cont_url])){
$collected_error=reset($this->exclude_cont_urls[$md5_cont_url]);
$collected_error=$this->exclude_url_msg($collected_error);
}
2.0
2019-02-19 09:52:13 +00:00
}
2.2
2019-06-23 02:20:58 +00:00
if(empty($collected_error)){
if(!empty($this->config['field_title'])){
$collected_data['title']=$field_vals[$this->config['field_title']]['value'];
}
if(!empty($collected_data['title'])){
2.0
2019-02-19 09:52:13 +00:00
2.2
2019-06-23 02:20:58 +00:00
if($mcollected->getCountByTitle($collected_data['title'])>0){
$collected_error='标题重复：'.mb_substr($collected_data['title'],0,300,'utf-8');
}
2.0
2019-02-19 09:52:13 +00:00
}
}
if(empty($collected_error)){
if($is_real_time){
$GLOBALS['real_time_release']->export(array($collected_data));
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
unset($collected_data['fields']);
unset($collected_data['title']);
}
$this->collected_field_list[]=$collected_data;
}else{
2.2
2019-06-23 02:20:58 +00:00
if(!$this->config['url_repeat']){
controller('ReleaseBase','event')->record_collected($collected_data['url'],
array('id'=>0,'error'=>$collected_error),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module'])
);
}else{
$this->echo_msg($collected_error);
}
2.0
2019-02-19 09:52:13 +00:00
}
}
}
if($is_loop){
controller('ReleaseBase','event')->record_collected(
2.2
2019-06-23 02:20:58 +00:00
$cont_url,array('id'=>1,'target'=>'','desc'=>'循环入库'),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module']),null,false
2.0
2019-02-19 09:52:13 +00:00
);
}
2.2
2019-06-23 02:20:58 +00:00
}else{
$this->echo_msg('已采集过该<a href="'.$cont_url.'" target="_blank">网址</a>','black');
2.0
2019-02-19 09:52:13 +00:00
}
$this->used_cont_urls[$md5_cont_url]=1;
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($this->collect_num>0){
if(count($this->collected_field_list)>=$this->collect_num){
if($cur_c_i<count($cont_urls)){
$finished_cont=false;
}
break;
}
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($finished_cont){
if($source_type==1){
$mcacheSource->setCache(md5($cont_key),$cont_key);
}elseif($source_type==2){
$mcacheLevel->setCache(md5($cont_key),$cont_key);
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($source_type==2){
$this->used_level_urls[$cont_key]=1;
}else{
$this->used_source_urls[$cont_key]=1;
}
}
2.2
2019-06-23 02:20:58 +00:00
2.0
2019-02-19 09:52:13 +00:00
if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){
break;
}
}
}
}
?>