show_opened_tools){ $opened_tools=array(); if($this->config['page_render']){ $opened_tools[]='页面渲染'; } if($GLOBALS['config']['caiji']['download_img']){ $opened_tools[]='图片本地化'; } if($GLOBALS['config']['proxy']['open']){ $opened_tools[]='代理'; } if(!empty($opened_tools)){ $this->echo_msg('开启功能：'.implode('、', $opened_tools),'black'); } if($num>0){ $this->echo_msg('预计采集'.$num.'条数据','black'); } $this->show_opened_tools=true; } $this->collect_num=$num; $this->collected_field_list=array(); $source_is_url=intval($this->config['source_is_url']); if(!isset($this->original_source_urls)){ $this->original_source_urls=array(); foreach ( $this->config ['source_url'] as $k => $v ) { if(empty($v)){ continue; } $return_s_urls = $this->convert_source_url ( $v ); if (is_array ( $return_s_urls )) { foreach ($return_s_urls as $r_s_u){ $this->original_source_urls[md5($r_s_u)]=$r_s_u; } } else { $this->original_source_urls[md5($return_s_urls)]=$return_s_urls; } } } if(empty($this->original_source_urls)){ $this->echo_msg('没有起始页网址！'); return 'completed'; } if($source_is_url){ if(isset($this->used_source_urls['_source_is_url_'])){ $this->echo_msg('所有起始页采集完毕！','green'); return 'completed'; } }else{ if(count($this->original_source_urls)<=count($this->used_source_urls)){ $this->echo_msg('所有起始页采集完毕！','green'); return 'completed'; } } $source_interval=$GLOBALS['config']['caiji']['interval']*60; $time_interval_list=array(); $source_urls=array(); $mcacheSource=CacheModel::getInstance('source_url'); if($source_is_url){ $source_urls=$this->original_source_urls; }else{ $cacheSources=$mcacheSource->db()->where(array('cname'=>array('in',array_keys($this->original_source_urls))))->column('dateline','cname'); if(!empty($cacheSources)){ $count_db_used=0; $sortSources=array('undb'=>array(),'db'=>array()); foreach ($this->original_source_urls as $sKey=>$sVal){ if(!isset($cacheSources[$sKey])){ $sortSources['undb'][$sKey]=$sVal; }else{ $time_interval=abs(NOW_TIME-$cacheSources[$sKey]); if($time_interval<$source_interval){ $this->used_source_urls[$sVal]=1; $count_db_used++; $time_interval_list[]=$time_interval; }else{ $sortSources['db'][$sKey]=$sVal; } } } if($count_db_used>0){ $this->echo_msg($count_db_used.'条已采集起始网址被过滤，下次采集需等待'.($source_interval-max($time_interval_list)).'秒，设置间隔','black'); if(count($this->original_source_urls)<=count($this->used_source_urls)){ $this->echo_msg('所有起始页采集完毕！','green'); return 'completed'; } } $source_urls=array_merge($sortSources['undb'],$sortSources['db']); unset($sortSources); unset($cacheSources); }else{ $source_urls=$this->original_source_urls; } } $mcollected=model('Collected'); if($source_is_url){ $this->cont_urls_list['_source_is_url_']=array_values($source_urls); $source_urls=array('_source_is_url_'=>'_source_is_url_'); } foreach ($source_urls as $key_source_url=>$source_url){ $this->cur_source_url=$source_url; if(array_key_exists($source_url,$this->used_source_urls)){ continue; } if($source_is_url){ $this->echo_msg("起始页已转换为内容页网址",'black'); }else{ $this->echo_msg("采集起始页：{$source_url}",'green'); } if($source_is_url){ $this->_collect_fields(); }else{ if(!empty($this->config['level_urls'])){ $this->echo_msg('开始分析多级网址','black'); $return_msg=$this->_collect_level($source_url,1); if($return_msg=='completed'){ return $return_msg; } }else{ $cont_urls=$this->getContUrls($source_url); $this->cont_urls_list[$source_url]=$this->_collect_unused_cont_urls($cont_urls); $this->_collect_fields(); } } if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){ break; } } return $this->collected_field_list; } /** * 优化设置页面post过来的config * @param unknown $config */ public function setConfig($config){ $config['url_complete']=intval($config['url_complete']); $config['url_reverse']=intval($config['url_reverse']); $config['page_render']=intval($config['page_render']); $config['url_repeat']=intval($config['url_repeat']); if(!empty($config['request_headers'])){ if(!is_array($config['request_headers']['custom_names'])){ $config['request_headers']['custom_names']=array(); } if(!is_array($config['request_headers']['custom_vals'])){ $config['request_headers']['custom_vals']=array(); } foreach ($config['request_headers']['custom_names'] as $k=>$v){ if(empty($v)){ unset($config['request_headers']['custom_names'][$k]); unset($config['request_headers']['custom_vals'][$k]); } } $config['request_headers']['custom_names']=array_values($config['request_headers']['custom_names']); $config['request_headers']['custom_vals']=array_values($config['request_headers']['custom_vals']); } foreach ($config['source_url'] as $k=>$v){ if(preg_match('/[\r\n]/', $v)){ if(preg_match_all('/^\w+\:\/\/[^\r\n]+/im',$v,$v_urls)){ $v_urls=array_unique($v_urls[0]); $v_urls=array_values($v_urls); $config['source_url'][$k]=implode("\r\n", $v_urls); }else{ unset($config['source_url'][$k]); } }else{ if(!preg_match('/^\w+\:\/\/.+/i', $v)){ unset($config['source_url'][$k]); } } } $config['source_url']=array_unique($config['source_url']); $config['source_url']=array_filter($config['source_url']); $config['source_url']=array_values($config['source_url']); if(!empty($config['field_list'])){ foreach ($config['field_list'] as $k=>$v){ $config['field_list'][$k]=json_decode(url_b64decode($v),true); } } if(!empty($config['field_process'])){ foreach ($config['field_process'] as $k=>$v){ $config['field_process'][$k]=json_decode(url_b64decode($v),true); $config['field_process'][$k]=$this->setProcess($config['field_process'][$k]); } } $config['common_process']=input('process/a',null,'trim'); $config['common_process']=$this->setProcess($config['common_process']); if(!empty($config['paging_fields'])){ foreach ($config['paging_fields'] as $k=>$v){ $config['paging_fields'][$k]=json_decode(url_b64decode($v),true); } } if(!empty($config['level_urls'])){ foreach ($config['level_urls'] as $k=>$v){ $config['level_urls'][$k]=json_decode(url_b64decode($v),true); } } if(!empty($config['relation_urls'])){ foreach ($config['relation_urls'] as $k=>$v){ $config['relation_urls'][$k]=json_decode(url_b64decode($v),true); } } $config['url_post']=intval($config['url_post']); if(!empty($config['url_posts'])){ if(!is_array($config['url_posts']['names'])){ $config['url_posts']['names']=array(); } if(!is_array($config['url_posts']['vals'])){ $config['url_posts']['vals']=array(); } if(!empty($config['url_posts']['names'])){ foreach ($config['url_posts']['names'] as $k=>$v){ if(empty($v)){ unset($config['url_posts']['names'][$k]); unset($config['url_posts']['vals'][$k]); } } } $config['url_posts']['names']=array_values($config['url_posts']['names']); $config['url_posts']['vals']=array_values($config['url_posts']['vals']); } return $config; } public function init($collData){ $collData['config']=unserialize($collData['config']); $this->collector=$collData; $releData=model('Release')->where(array('task_id'=>$collData['task_id']))->find(); if(!empty($releData)){ $releData=$releData->toArray(); } $this->release=$releData; $keyConfig='collector_config_'.$collData['id']; $cacheConfig=cache($keyConfig); if(empty($cacheConfig)||$cacheConfig['update_time']!=$collData['uptime']){ $config=$this->initConfig($collData['config']); cache($keyConfig,array('update_time'=>$collData['uptime'],'config'=>$config)); }else{ $config=$cacheConfig['config']; } $this->config=$config; } public function initConfig($config){ $newConfig=array(); $newConfig['charset'] = $config['charset']=='custom' ? $config ['charset_custom'] : $config ['charset']; $newConfig['charset']= empty($newConfig['charset'])?'auto':$newConfig['charset']; if(!empty($config['area'])){ if(empty($config['area_module'])){ $newConfig['reg_source_cont']=$this->convert_sign_match($config['area']); } }elseif(!empty($config['area_start'])||!empty($config['area_end'])) { $newConfig['reg_source_cont'] = $config['area_start'] . (!empty($config['area_end']) ? '(?P[\s\S]+?)' : '(?P[\s\S]+)') . $config['area_end']; } if(!empty($newConfig['reg_source_cont'])){ $newConfig['reg_source_cont'] = str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont'] ); $newConfig['reg_source_cont'] = preg_replace ( '/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont'] ); }elseif(!empty($config['area_module'])){ $newConfig['reg_source_cont']=$config['area']; } if(empty($config['url_rule_module'])){ if(!empty($config['url_rule'])){ $newConfig['reg_source_cont_url']=$this->convert_sign_match($config['url_rule']); $newConfig['reg_source_cont_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont_url']); $newConfig['reg_source_cont_url']=str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont_url'] ); }else{ $newConfig['reg_source_cont_url']='\bhref=[\'\"](?P[^\'\"\<\>]+?)[\'\"]'; } $config['url_merge']=$this->set_merge_default($newConfig['reg_source_cont_url'], $config['url_merge']); }elseif('xpath'==$config['url_rule_module']){ if(!empty($config['url_rule'])){ $newConfig['reg_source_cont_url']=$config['url_rule']; }else{ $newConfig['reg_source_cont_url']='//a'; } $config['url_merge']=$this->set_merge_default('(?P.+)', $config['url_merge']); }elseif('json'==$config['url_rule_module']){ $newConfig['reg_source_cont_url']=$config['url_rule']; $config['url_merge']=$this->set_merge_default('(?P.+)', $config['url_merge']); } if(!empty($config['url_must'])){ $newConfig['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_must']); $newConfig['url_must']=str_replace('(*)', '[\s\S]*?', $newConfig['url_must']); } if(!empty($config['url_ban'])){ $newConfig['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_ban']); $newConfig['url_ban']=str_replace('(*)', '[\s\S]*?', $newConfig['url_ban']); } if(!empty($config['level_urls'])){ $config['new_level_urls']=array(); foreach ($config['level_urls'] as $luk=>$luv){ if(!empty($luv['area'])){ if(empty($luv['area_module'])){ $luv['reg_area']=$this->convert_sign_match($luv['area']); $luv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_area']); $luv['reg_area']=str_replace('(*)', '[\s\S]*?', $luv['reg_area']); }else{ $luv['reg_area']=$luv['area']; } $luv['reg_area_module']=$luv['area_module']; } if(empty($luv['url_rule_module'])){ if(!empty($luv['url_rule'])){ $luv['reg_url']=$this->convert_sign_match($luv['url_rule']); $luv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_url']); $luv['reg_url']=str_replace ( '(*)', '[\s\S]*?', $luv['reg_url'] ); }else{ $luv['reg_url']='\bhref=[\'\"](?P[^\'\"\<\>]+?)[\'\"]'; } $luv['url_merge']=$this->set_merge_default($luv['reg_url'], $luv['url_merge']); }elseif('xpath'==$luv['url_rule_module']){ if(!empty($luv['url_rule'])){ $luv['reg_url']=$luv['url_rule']; }else{ $luv['reg_url']='//a'; } $luv['url_merge']=$this->set_merge_default('(?P.+)', $luv['url_merge']); }elseif('json'==$luv['url_rule_module']){ $luv['reg_url']=$luv['url_rule']; $luv['url_merge']=$this->set_merge_default('(?P.+)', $luv['url_merge']); } $luv['reg_url_module']=$luv['url_rule_module']; if(!empty($luv['url_must'])){ $luv['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_must']); $luv['url_must']=str_replace('(*)', '[\s\S]*?', $luv['url_must']); } if(!empty($luv['url_ban'])){ $luv['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_ban']); $luv['url_ban']=str_replace('(*)', '[\s\S]*?', $luv['url_ban']); } $config['level_urls'][$luk]=$luv; $config['new_level_urls'][$luv['name']]=$luv; } } $relation_urls=array(); if(!empty($config['relation_urls'])){ foreach ($config['relation_urls'] as $ruv){ if(!empty($ruv['area'])){ if(empty($ruv['area_module'])){ $ruv['reg_area']=$this->convert_sign_match($ruv['area']); $ruv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_area']); $ruv['reg_area']=str_replace('(*)', '[\s\S]*?', $ruv['reg_area']); }else{ $ruv['reg_area']=$ruv['area']; } $ruv['reg_area_module']=$ruv['area_module']; } if(empty($ruv['url_rule_module'])){ $ruv['reg_url']=$this->convert_sign_match($ruv['url_rule']); $ruv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_url']); $ruv['reg_url']=str_replace('(*)', '[\s\S]*?', $ruv['reg_url']); $ruv['url_merge']=$this->set_merge_default($ruv['reg_url'], $ruv['url_merge']); }elseif(in_array($ruv['url_rule_module'],array('xpath','json'))){ $ruv['reg_url']=$ruv['url_rule']; $ruv['url_merge']=$this->set_merge_default('(?P.+)', $ruv['url_merge']); } $ruv['reg_url_module']=$ruv['url_rule_module']; $relation_urls[$ruv['name']]=$ruv; } } $relation_depth_urls=array(); foreach ($relation_urls as $ruv){ $rDepth=0; $rFuName=$ruv['page']; if(empty($rFuName)){ $rDepth=0; }else{ $passRelation=false; $rFuPage=$rFuName; do{ if(empty($relation_urls[$rFuPage])){ $passRelation=true; break; } $rFuPage=$relation_urls[$rFuPage]['page']; if($rFuPage==$rFuName){ $passRelation=true; break; } $rDepth++; }while(!empty($rFuPage)); if($passRelation){ continue; } } $relation_depth_urls[$rDepth][$ruv['name']]=$ruv; } ksort($relation_depth_urls); $config['new_relation_urls']=array(); foreach ($relation_depth_urls as $rurls){ if(is_array($rurls)){ $config['new_relation_urls']=array_merge($config['new_relation_urls'],$rurls); } } if(!empty($config['field_list'])){ foreach ($config['field_list'] as $fk=>$fv){ if('rule'==$fv['module']){ $fv['reg_rule']=$this->convert_sign_match($fv['rule']); $fv['reg_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $fv['reg_rule']); $fv['reg_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_rule']); $fv['rule_merge']=$this->set_merge_default($fv['reg_rule'], $fv['rule_merge']); }elseif('extract'==$fv['module']){ if(!empty($fv['extract_rule'])){ $fv['reg_extract_rule']=$this->convert_sign_match($fv['extract_rule']); $fv['reg_extract_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1",$fv['reg_extract_rule']); $fv['reg_extract_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_extract_rule']); $fv['extract_rule_merge']=$this->set_merge_default($fv['reg_extract_rule'], ''); } } $config['field_list'][$fk]=$fv; } } if(!empty($config['field_process'])){ foreach ($config['field_process'] as $k=>$v){ $config['field_process'][$k]=$this->initProcess($v); } } if(!empty($config['common_process'])){ $config['common_process']=$this->initProcess($config['common_process']); } if(!empty($config['paging']['area'])){ if(empty($config['paging']['area_module'])){ $config['paging']['reg_area']=$this->convert_sign_match($config['paging']['area']); $config['paging']['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_area']); $config['paging']['reg_area']=str_replace('(*)', '[\s\S]*?', $config['paging']['reg_area']); $config['paging']['reg_area_merge']=$this->set_merge_default($config['paging']['reg_area'], ''); }else{ $config['paging']['reg_area']=$config['paging']['area']; } $config['paging']['reg_area_module']=$config['paging']['area_module']; } if(!empty($config['paging']['url_rule'])){ if(empty($config['paging']['url_rule_module'])){ $config['paging']['reg_url']=$this->convert_sign_match($config['paging']['url_rule']); $config['paging']['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_url']); $config['paging']['reg_url']=str_replace ( '(*)', '[\s\S]*?', $config['paging']['reg_url'] ); $config['paging']['url_merge']=$this->set_merge_default($config['paging']['reg_url'], $config['paging']['url_merge']); if(empty($config['paging']['url_merge'])){ $config['paging']['url_merge']=cp_sign('match'); } }else{ $config['paging']['reg_url']=$config['paging']['url_rule']; $config['paging']['url_merge']=$this->set_merge_default('(?P.+)', $config['paging']['url_merge']); } $config['paging']['reg_url_module']=$config['paging']['url_rule_module']; } if(!empty($config['paging']['url_must'])){ $config['paging']['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_must']); $config['paging']['url_must']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_must']); } if(!empty($config['paging']['url_ban'])){ $config['paging']['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_ban']); $config['paging']['url_ban']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_ban']); } $module_normal_fields=array(); $module_extract_fields=array(); $module_merge_fields=array(); if(!empty($config['field_list'])){ foreach ($config['field_list'] as $fk=>$fv){ $fieldModule=strtolower($fv['module']); $fieldConfig=array('field'=>$fv,'process'=>$config['field_process'][$fk]); if('extract'==$fieldModule){ $module_extract_fields[$fv['name']]=$fieldConfig; }elseif('merge'==$fieldModule){ $module_merge_fields[$fv['name']]=$fieldConfig; }else{ $module_normal_fields[$fv['name']]=$fieldConfig; } } } $config['new_field_list']=array_merge($module_normal_fields,$module_extract_fields,$module_merge_fields); $new_paging_fields=array( 'normal'=>array(), 'extract'=>array(), 'merge'=>array(), ); if(!empty($config['paging_fields'])){ $pagingFields=array(); foreach ($config['paging_fields'] as $pfield){ $pagingFields[$pfield['field']]=$pfield; } if(!empty($pagingFields['::all'])){ $fieldAllParams=$pagingFields['::all']; unset($pagingFields['::all']); foreach ($config['new_field_list'] as $k=>$v){ if(empty($pagingFields[$k])){ $fieldAllParams['field']=$k; $pagingFields[$k]=$fieldAllParams; } } } $config['paging_fields']=$pagingFields; unset($pagingFields); foreach ($config['paging_fields'] as $pfk=>$pfield){ $pfield['delimiter']=str_replace(array('\r','\n'), array("\r","\n"), $pfield['delimiter']); $config['paging_fields'][$pfk]=$pfield; if(!empty($module_normal_fields[$pfield['field']])){ $new_paging_fields['normal'][$pfield['field']]=$pfield; }elseif(!empty($module_extract_fields[$pfield['field']])){ $new_paging_fields['extract'][$pfield['field']]=$pfield; }elseif(!empty($module_merge_fields[$pfield['field']])){ $new_paging_fields['merge'][$pfield['field']]=$pfield; } } } $config['new_paging_fields']=array_merge($new_paging_fields['normal'],$new_paging_fields['extract'],$new_paging_fields['merge']); $config=array_merge($config,$newConfig); return $config; } /*采集级别网址*/ public function collLevelUrls($source_url,$curLevel=1){ $curLevel=$curLevel>0?$curLevel:0; if($curLevel>0){ $nextLevel=0; if(!empty($this->config['level_urls'])){ if(!empty($this->config['level_urls'][$curLevel-1])){ if(!empty($this->config['level_urls'][$curLevel])){ $nextLevel=$curLevel+1; } } } $cont_urls=$this->getLevelUrls($source_url,$curLevel); }else{ $cont_urls=$this->getContUrls($source_url); } return array('urls'=>$cont_urls,'levelName'=>$this->config['level_urls'][$curLevel-1]['name'],'nextLevel'=>$nextLevel); } /*获取内容网址*/ public function getContUrls($source_url){ if(empty($source_url)){ return $this->error('请输入起始网址'); } $config=array( 'reg_area'=>$this->config['reg_source_cont'], 'reg_area_module'=>$this->config['area_module'], 'reg_url'=>$this->config['reg_source_cont_url'], 'reg_url_module'=>$this->config['url_rule_module'], 'url_merge'=>$this->config['url_merge'], 'url_must'=>$this->config['url_must'], 'url_ban'=>$this->config['url_ban'], ); return $this->_get_urls($source_url, $config); } /*获取多级网址*/ public function getLevelUrls($parent_url,$level=1){ $level=$level>1?$level:1; $config=$this->config['level_urls'][$level-1]; if(empty($config)){ return $this->error('没有'.($level).'级网址规则'); } if(empty($config['reg_url'])){ return $this->error('必须填写多级“提取网址规则”'); } if(empty($parent_url)){ return $this->error('请输入父级网址'); } return $this->_get_urls($parent_url, $config,true); } /*获取分页链接*/ public function getPagingUrls($from_url,$html,$is_test=false){ $paging_urls=array(); if($this->config['paging']['open']){ if(empty($html)){ $html=$this->get_html($from_url); } if(!empty($this->config['paging']['reg_url'])){ if(!empty($this->config['new_paging_fields'])){ $base_url=$this->match_base_url($from_url, $html); $domain_url=$this->match_domain_url($from_url); $paging_area=''; if(!empty($this->config['paging']['reg_area'])){ if(empty($this->config['paging']['reg_area_module'])){ $sign_match=$this->sign_addslashes(cp_sign('match','(?P\d*)')); if(preg_match_all('/'.$sign_match.'/i', $this->config['paging']['reg_area_merge'],$match_signs)){ if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){ $re_match=array(); foreach($match_signs['num'] as $ms_k=>$ms_v){ $re_match[$ms_k]=$m_paging_area['match'.$ms_v]; } $paging_area=str_replace($match_signs[0], $re_match, $this->config['paging']['reg_area_merge']); } }else{ if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){ $paging_area=$m_paging_area[0]; } } }elseif('json'==$this->config['paging']['reg_area_module']){ $paging_area=$this->rule_module_json_data(array('json'=>$this->config['paging']['reg_area'],'json_arr'=>'jsonencode'),$html); }elseif('xpath'==$this->config['paging']['reg_area_module']){ $paging_area=$this->rule_module_xpath_data(array('xpath'=>$this->config['paging']['reg_area'],'xpath_attr'=>'outerHtml'),$html); } }else{ $paging_area=$html; } if(!empty($paging_area)){ if(!empty($this->config['paging']['url_complete'])){ $paging_area=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche_p_a) use ($base_url,$domain_url){ return \skycaiji\admin\event\Cpattern::create_complete_url($matche_p_a[1], $base_url, $domain_url); },$paging_area); } $m_paging_urls=$this->rule_match_urls($this->config['paging'],$paging_area,true); foreach ($m_paging_urls as $purl){ if(!empty($this->config['paging']['url_must'])){ if(!preg_match('/'.$this->config['paging']['url_must'].'/i', $purl)){ continue; } } if(!empty($this->config['paging']['url_ban'])){ if(preg_match('/'.$this->config['paging']['url_ban'].'/i', $purl)){ continue; } } if($from_url==$purl){ continue; } if(strpos($purl,' ')==false){ $paging_urls[]=$purl; } } if(!empty($paging_urls)){ $paging_urls=array_filter($paging_urls); $paging_urls=array_unique($paging_urls); $paging_urls=array_values($paging_urls); }else{ if($is_test){ return $this->error('未获取到分页链接，请检查分页链接规则'); } } }else{ if($is_test){ return $this->error('未获取到分页区域，请检查分页区域规则'); } } }else{ if($is_test){ return $this->error('请添加分页内容字段'); } } }else{ if($is_test){ return $this->error('必须填写分页链接规则'); } } }else{ if($is_test){ return $this->error('未开启分页'); } } return $paging_urls; } /*设置字段值*/ public function setField($field_config,$cur_url,$html,$cont_url){ $cur_url_md5=md5($cur_url); $field_process=$field_config['process']; $field_params=$field_config['field']; $module=strtolower($field_params['module']); if(!empty($field_params['source'])&&in_array($module, array('rule','xpath','json','auto'))){ $field_source_url=''; $source_echo_msg='——采集'; if('source_url'==$field_params['source']){ $field_source_url=$this->cur_source_url; $source_echo_msg.='起始页'; }elseif(preg_match('/^relation_url:(.+)$/i', $field_params['source'],$relationName)){ $relationName=$relationName[1]; $field_source_url=$this->getRelationUrl($relationName, $cur_url, $html); $source_echo_msg.="关联页“{$relationName}”"; }elseif(preg_match('/^level_url:(.+)$/i', $field_params['source'],$levelName)){ $levelName=$levelName[1]; if(empty($this->config['new_level_urls'][$levelName])){ return; } if(empty($this->cur_level_urls[$levelName])){ return; } $field_source_url=$this->cur_level_urls[$levelName]; $source_echo_msg.="多级页“{$levelName}”"; } if(empty($field_source_url)){ return; } if($field_source_url!=$cur_url){ $cur_url=$field_source_url; $this->echo_msg($source_echo_msg."：{$field_source_url}",'black'); $html=$this->get_html($field_source_url,true); } } static $fieldArr1=array('words','num','time','list'); static $fieldArr2=array('auto','json'); static $baseUrls=array(); static $domainUrls=array(); $urlMd5=md5($cur_url); if(empty($baseUrls[$urlMd5])){ $baseUrls[$urlMd5]=$this->match_base_url($cur_url, $html); } if(empty($domainUrls[$urlMd5])){ $domainUrls[$urlMd5]=$this->match_domain_url($cur_url); } $base_url=$baseUrls[$urlMd5]; $domain_url=$domainUrls[$urlMd5]; $val=''; $field_func='field_module_'.$module; if(method_exists($this, $field_func)){ if('extract'==$module){ if(is_array($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5])){ $val=array(); foreach ($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5] as $k=>$v){ $extract_field_val=array( 'value'=>$v, 'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5][$k], ); $val[$k]=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url); } }else{ $extract_field_val=array( 'value'=>$this->field_val_list[$field_params['extract']]['values'][$cur_url_md5], 'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5], ); $val=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url); } }elseif('merge'==$module){ if(empty($this->first_loop_field)){ $cur_field_val_list=array(); foreach ($this->field_val_list as $k=>$v){ $cur_field_val_list[$k]=array( 'value'=>$v['values'][$cur_url_md5], 'img'=>$v['imgs'][$cur_url_md5] ); } $val=$this->field_module_merge($field_params,$cur_field_val_list); }else{ $val=array(); foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){ $cur_field_val_list=array(); foreach ($this->field_val_list as $k=>$v){ $cur_field_val_list[$k]=array( 'value'=>(is_array($v['values'][$cur_url_md5])?$v['values'][$cur_url_md5][$v_k]:$v['values'][$cur_url_md5]), 'img'=>(is_array($v['imgs'][$cur_url_md5][$v_k])?$v['imgs'][$cur_url_md5][$v_k]:$v['imgs'][$cur_url_md5]) ); } $val[$v_k]=$this->field_module_merge($field_params,$cur_field_val_list); } } }elseif(in_array($module,$fieldArr1)){ if($module=='words'){ $val=$this->$field_func($field_params); }else{ if(empty($this->first_loop_field)){ $val=$this->$field_func($field_params); }else{ $val=array(); foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){ $val[$v_k]=$this->$field_func($field_params); } } } }elseif(in_array($module,$fieldArr2)){ $val=$this->$field_func($field_params,$html,$cur_url); }else{ $val=$this->$field_func($field_params,$html); } } $vals=null; if(is_array($val)){ $is_loop=true; $vals=array_values($val); }else{ $is_loop=false; $vals=array($val); } $field_name=$field_params['name']; if(!isset($this->field_val_list[$field_name])){ $this->field_val_list[$field_name]=array('values'=>array(),'imgs'=>array()); } $cont_url_md5=empty($cont_url)?$cur_url_md5:md5($cont_url); foreach ($vals as $v_k=>$val){ $loopIndex=$is_loop?$v_k:-1; if(!empty($field_process)){ $val=$this->process_field($val,$field_process,$cur_url_md5,$loopIndex,$cont_url_md5); } if(!empty($this->config['common_process'])){ $val=$this->process_field($val,$this->config['common_process'],$cur_url_md5,$loopIndex,$cont_url_md5); } if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5])){ if(empty($this->first_loop_field)){ foreach ($this->field_val_list as $f_k=>$f_v){ unset($this->field_val_list[$f_k]['values'][$cur_url_md5]); unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]); } return; }else{ if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5][$loopIndex])){ if(!$is_loop){ foreach ($this->field_val_list as $f_k=>$f_v){ unset($this->field_val_list[$f_k]['values'][$cur_url_md5]); unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]); } return; }else{ foreach ($this->field_val_list as $f_k=>$f_v){ if(is_array($this->field_val_list[$f_k]['values'][$cur_url_md5])){ unset($this->field_val_list[$f_k]['values'][$cur_url_md5][$v_k]); } if(is_array($this->field_val_list[$f_k]['imgs'][$cur_url_md5])){ unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5][$v_k]); } } continue; } } } } $val=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){ return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url); },$val); $val=preg_replace_callback('/(?<=\bsrc\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){ return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url); },$val); if($is_loop){ if(!isset($this->field_val_list[$field_name]['values'][$cur_url_md5])){ $this->field_val_list[$field_name]['values'][$cur_url_md5]=array(); $this->field_val_list[$field_name]['imgs'][$cur_url_md5]=array(); } $this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$val; }else{ $this->field_val_list[$field_name]['values'][$cur_url_md5]=$val; } if(!empty($GLOBALS['config']['caiji']['download_img'])&&!empty($val)){ $valImgs=array(); if(preg_match_all('/]*\bsrc=[\'\"]*(\w+\:\/\/[^\'\"\s]+)[\'\"]*/i',$val,$imgUrls)){ $valImgs=is_array($imgUrls[1])?$imgUrls[1]:array(); } if('extract'==$module&&'cover'==$field_params['extract_module']){ $valImgs=array_merge($valImgs,array($val)); } $noImgVal=preg_replace_callback('/\{\[img\]\}(http[s]{0,1}\:\/\/[^\s]+?)\{\[\/img\]\}/i',function($matche) use (&$valImgs){ $valImgs[]=$matche[1]; return $matche[1]; },$val); if($noImgVal!=$val){ if($is_loop){ $this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$noImgVal; }else{ $this->field_val_list[$field_name]['values'][$cur_url_md5]=$noImgVal; } } if(!empty($valImgs)){ $valImgs=array_unique($valImgs); $valImgs=array_values($valImgs); if($is_loop){ $this->field_val_list[$field_name]['imgs'][$cur_url_md5][$v_k]=$valImgs; }else{ $this->field_val_list[$field_name]['imgs'][$cur_url_md5]=$valImgs; } } } } } /*设置分页的字段列表值*/ public function setPagingFields($cont_url,$page_url){ $contMd5=md5($cont_url); $pageMd5=md5($page_url); if(empty($page_url)){ return $this->error('请输入分页网址'); } if(!preg_match('/^\w+\:\/\//',$page_url)){ return $this->error($page_url.'网址不完整'); } if(empty($this->config['paging']['max'])||(count($this->used_paging_urls[$contMd5])<$this->config['paging']['max'])){ $this->set_html_interval(); $this->echo_msg("——采集分页：{$page_url}",'black'); $html=$this->get_html($page_url); if(empty($html)){ return $this->error('分页获取失败：'.$page_url); } if(!isset($this->used_paging_urls[$contMd5][$pageMd5])){ $this->used_paging_urls[$contMd5][$pageMd5]=$page_url; foreach ($this->config['new_paging_fields'] as $v){ $this->setField($this->config['new_field_list'][$v['field']],$page_url,$html,$cont_url); } } $paging_urls=$this->getPagingUrls($page_url,$html); if(!empty($paging_urls)){ $nextUrl=''; foreach ($paging_urls as $purl){ if(!isset($this->used_paging_urls[$contMd5][md5($purl)])&&$cont_url!=$purl){ $nextUrl=$purl; break; } } if(!empty($nextUrl)){ $this->setPagingFields($cont_url,$nextUrl); } } } } /** * 获取关联页网址 * @param unknown $name 关联页名称 * @param unknown $cont_url 内容页网址 * @param unknown $html 内容页源码 * @return string */ public function getRelationUrl($name,$cont_url,$html){ if(empty($html)){ $html=$this->get_html($cont_url,true); } if(empty($html)){ return ''; } $relation_url=$this->config['new_relation_urls'][$name]; if(empty($relation_url)){ return ''; } if(empty($relation_url['page'])){ if(!isset($this->relation_url_list[$cont_url][$name])){ $html=$this->rule_match_area($relation_url, $html); if(empty($html)){ return ''; } $relationUrl=$this->rule_match_urls($relation_url, $html); $relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):''; $this->relation_url_list[$cont_url][$name]=$relationUrl; }else{ $relationUrl=$this->relation_url_list[$cont_url][$name]; } }else{ $page=$relation_url['page']; $pass=false; $depth_pages=array(); $depth=0; while(!empty($page)){ if($page==$name){ $pass=true; break; } if(empty($this->config['new_relation_urls'][$page])){ $pass=true; break; } $depth++; $depth_pages[$depth]=$page; $page=$this->config['new_relation_urls'][$page]['page']; } if($pass){ return ''; } krsort($depth_pages); $contPage=reset($depth_pages); $relationUrl=''; if(isset($contPage)){ if(!isset($this->relation_url_list[$cont_url][$contPage])){ $html=$this->rule_match_area($this->config['new_relation_urls'][$contPage], $html); if(empty($html)){ return ''; } $relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$contPage], $html); $relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):''; $this->relation_url_list[$cont_url][$contPage]=$relationUrl; }else{ $relationUrl=$this->relation_url_list[$cont_url][$contPage]; } } $depth_pages=array_slice($depth_pages, 1); $depth_pages=is_array($depth_pages)?$depth_pages:array(); $depth_pages[]=$relation_url['name']; foreach ($depth_pages as $page){ if(empty($relationUrl)){ return ''; } if(!isset($this->relation_url_list[$cont_url][$page])){ $relationHtml=$this->get_html($relationUrl,true); $relationHtml=$this->rule_match_area($this->config['new_relation_urls'][$page], $relationHtml); if(empty($relationHtml)){ return ''; } $relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$page],$relationHtml); $relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):''; $this->relation_url_list[$cont_url][$page]=$relationUrl; }else{ $relationUrl=$this->relation_url_list[$cont_url][$page]; } } } return $relationUrl; } /*获取内容页字段列表，这里是入口*/ public function getFields($cont_url){ $this->field_val_list=array(); $this->first_loop_field=null; if(empty($cont_url)){ return $this->error('请输入内容页网址'); } if(!preg_match('/^\w+\:\/\//',$cont_url)){ return $this->error($cont_url.'网址不完整'); } $html=$this->get_html($cont_url,false,$this->config['url_post']); if(empty($html)){ return $this->error('抓取页面失败'); } foreach($this->config['new_field_list'] as $field_config){ $this->setField($field_config,$cont_url,$html,$cont_url); } $paging_urls=$this->getPagingUrls($cont_url,$html); if(!empty($paging_urls)){ $this->setPagingFields($cont_url,reset($paging_urls)); } $val_list=array(); if(!empty($this->field_val_list)){ if(empty($this->first_loop_field)){ foreach ($this->field_val_list as $fieldName=>$fieldVal){ $val_values=array_filter($fieldVal['values']); $val_values=implode($this->config['new_paging_fields'][$fieldName]['delimiter'], $val_values); $val_imgs=array(); if(!empty($fieldVal['imgs'])){ foreach ($fieldVal['imgs'] as $v){ if(!empty($v)){ if(is_array($v)){ $val_imgs=array_merge($val_imgs,$v); }else{ $val_imgs[]=$v; } } } if(!empty($val_imgs)){ $val_imgs=array_unique($val_imgs); $val_imgs=array_filter($val_imgs); $val_imgs=array_values($val_imgs); } } $val_list[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs); } }else{ foreach ($this->field_val_list[$this->first_loop_field]['values'] as $page_key=>$page_vals){ if(empty($page_vals)){ continue; } foreach ($page_vals as $loop_index=>$loop_val){ $vals=array(); foreach ($this->field_val_list as $fieldName=>$fieldVals){ if(is_array($fieldVals['values'][$page_key])){ $val_values=$fieldVals['values'][$page_key][$loop_index]; $val_imgs=$fieldVals['imgs'][$page_key][$loop_index]; }else{ $val_values=$fieldVals['values'][$page_key]; $val_imgs=$fieldVals['imgs'][$page_key]; } if(!empty($val_imgs)){ $val_imgs=array_unique($val_imgs); $val_imgs=array_filter($val_imgs); $val_imgs=array_values($val_imgs); } $vals[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs); } $val_list[]=$vals; } } } } return $val_list?$val_list:array(); } /*设置数据处理，保存config时使用*/ public function setProcess($processList){ if(is_array($processList)){ $processList=array_array_map('trim',$processList); foreach ($processList as $k=>$v){ $v['module']=strtolower($v['module']); if(!empty($v['title'])){ $v['title']=str_replace(array("'",'"'),'',strip_tags($v['title'])); } if('html'==$v['module']){ $v['html_allow']=$this->clear_tags($v['html_allow']); $v['html_allow']=implode(',', $v['html_allow']); $v['html_filter']=$this->clear_tags($v['html_filter']); $v['html_filter']=implode(',', $v['html_filter']); }elseif('filter'==$v['module']){ if(preg_match_all('/[^\r\n]+/', $v['filter_list'],$filterList)){ $filterList=array_filter(array_unique($filterList[0])); $v['filter_list']=implode("\r\n",$filterList); } $v['filter_list']=trim($v['filter_list']); } $processList[$k]=$v; } $processList=array_values($processList); } return $processList; } /*初始化数据处理，初始化config时使用*/ public function initProcess($processList){ if(!empty($processList)){ foreach ($processList as $k=>$v){ if('replace'==$v['module']){ $v['replace_from']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $v['replace_from']); $v['replace_from']=str_replace('(*)', '[\s\S]*?', $v['replace_from']); } $processList[$k]=$v; } } return $processList; } /*统一：获取网址列表*/ public function _get_urls($source_url,$config,$is_level=false){ $is_level=$is_level?'多级':''; $html=$this->get_html($source_url); if(empty($html)){ return $this->error($is_level.'页面为空'); } $base_url=$this->match_base_url($source_url, $html); $domain_url=$this->match_domain_url($source_url); $html=$this->rule_match_area($config, $html); if(empty($html)){ return $this->error("未提取到{$is_level}区域内容！"); } $cont_urls=$this->rule_match_urls($config, $html); $cont_urls1=array(); if(isset($this->config['url_op'])){ $op_not_complete=in_array('not_complete',$this->config['url_op'])?true:false; }else{ if(isset($this->config['url_complete'])){ $op_not_complete=$this->config['url_complete']?false:true; }else{ $op_not_complete=false; } } foreach ($cont_urls as $cont_url){ if(!$op_not_complete){ $cont_url=$this->create_complete_url($cont_url, $base_url, $domain_url); } if(!empty($config['url_must'])){ if(!preg_match('/'.$config['url_must'].'/i', $cont_url)){ continue; } } if(!empty($config['url_ban'])){ if(preg_match('/'.$config['url_ban'].'/i', $cont_url)){ continue; } } if(!empty($cont_url)){ if(strpos($cont_url,' ')==false){ $cont_urls1[]=$cont_url; } } } $cont_urls=$cont_urls1; unset($cont_urls1); if(empty($cont_urls)){ return $this->error("未获取到".($is_level?$is_level:'内容')."网址！"); }else{ if(!empty($this->config['url_reverse'])){ $cont_urls=array_reverse($cont_urls); } if(!empty($this->config['url_post'])){ $postParams=array(); if(!empty($this->config['url_posts']['names'])){ foreach ($this->config['url_posts']['names'] as $k=>$v){ if (!empty($v)){ $postParams[]=$v.'='.rawurlencode($this->config['url_posts']['vals'][$k]); } } } if(!empty($postParams)){ $postParams=implode('&', $postParams); foreach ($cont_urls as $k=>$v){ $v.=strpos($v,'?')===false?'?':'&'; $v.=$postParams; $cont_urls[$k]=$v; } } } return array_values($cont_urls); } } /*执行采集返回未使用的网址*/ public function _collect_unused_cont_urls($cont_urls=array(),$echo_str=''){ $mcollected=model('Collected'); $count_conts=count($cont_urls); if($this->config['url_repeat']){ $db_cont_urls=array(); }else{ $db_cont_urls=$mcollected->getUrlByUrl($cont_urls); } $unused_cont_urls=array(); $count_used=0; if(!empty($cont_urls)){ foreach ($cont_urls as $cont_url){ if(array_key_exists(md5($cont_url), $this->used_cont_urls)){ $count_used++; }elseif(in_array($cont_url, $db_cont_urls)){ $count_used++; }else{ $unused_cont_urls[md5($cont_url)]=$cont_url; } } } if($count_used>0){ $count_used=min(count($cont_urls),$count_used); $this->echo_msg($echo_str.'采集到'.$count_conts.'条网址，'.$count_used.'条重复，'.(count($unused_cont_urls)).'条有效','black'); }else{ $this->echo_msg($echo_str.'采集到'.$count_conts.'条有效网址','black'); } return $unused_cont_urls; } /*执行级别采集*/ public function _collect_level($source_url,$level=1){ $end_echo=''; $level=max(1,$level); $level_str=''; for($i=1;$i<$level;$i++){ } $next_level_str=$level_str; if($level<=1){ $this->cur_level_urls=array(); } $this->echo_msg('','',true,'
'); $level_data=$this->collLevelUrls($source_url,$level); $this->echo_msg($level_str.'抓取到'.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址'.count($level_data['urls']).'条','black'); $mcollected=model('Collected'); $mcacheLevel=CacheModel::getInstance('level_url'); if(!empty($level_data['urls'])){ $level_urls=array(); foreach ($level_data['urls'] as $level_url){ $level_urls["level_{$level}:{$level_url}"]=$level_url; } $level_interval=$GLOBALS['config']['caiji']['interval']*60; $time_interval_list=array(); $cacheLevels=$mcacheLevel->db()->where(array('cname'=>array('in',array_map('md5', array_keys($level_urls)))))->column('dateline','cname'); if(!empty($cacheLevels)){ $count_db_used=0; $sortLevels=array('undb'=>array(),'db'=>array()); foreach ($level_urls as $level_key=>$level_url){ $md5_level_key=md5($level_key); if(!isset($cacheLevels[$md5_level_key])){ $sortLevels['undb'][$level_key]=$level_url; }else{ $time_interval=abs(NOW_TIME-$cacheLevels[$md5_level_key]); if($time_interval<$level_interval){ $this->used_level_urls[$level_key]=1; $count_db_used++; $time_interval_list[]=$time_interval; }else{ $sortLevels['db'][$level_key]=$level_url; } } } if($count_db_used>0){ $this->echo_msg($level_str.$count_db_used.'条已采集网址被过滤，下次采集需等待'.($level_interval-max($time_interval_list)).'秒，设置间隔','black'); if(count($level_urls)<=$count_db_used){ $this->echo_msg($level_str.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址采集完毕！','green',true,$end_echo); return 'completed'; } } $level_urls=array_merge($sortLevels['undb'],$sortLevels['db']); unset($sortLevels); unset($cacheLevels); } $level_data['urls']=$level_urls; } $finished_source=true; $cur_level_i=0;; if(!empty($level_data['urls'])){ foreach ($level_data['urls'] as $level_key=>$level_url){ $cur_level_i++; if(array_key_exists($level_key,$this->used_level_urls)){ continue; } $this->cur_level_urls[$this->config['level_urls'][$level-1]['name']]=$level_url; $this->echo_msg("{$next_level_str}分析第{$level}级：{$level_url}",'black'); if($level_data['nextLevel']>0){ $return_msg=$this->_collect_level($level_url,$level_data['nextLevel']); if($return_msg=='completed'){ $this->echo_msg('','',true,$end_echo); return $return_msg; } }else{ $cont_urls=$this->getContUrls($level_url); $cont_urls=$this->_collect_unused_cont_urls($cont_urls,$next_level_str); $this->cont_urls_list[$level_key]=$cont_urls; $this->_collect_fields($next_level_str); } if($this->collect_num>0){ if(count($this->collected_field_list)>=$this->collect_num){ if($cur_level_iused_level_urls[$source_key]=1; $mcacheLevel->setCache(md5($source_key),$source_key); if($level<=1){ $mcacheSource=CacheModel::getInstance('source_url'); $this->used_source_urls[$source_url]=1; $mcacheSource->setCache(md5($source_url),$source_url); } } $this->echo_msg('','',true,$end_echo); } /*采集字段列表*/ public function _collect_fields($echo_str=''){ $mcollected=model('Collected'); $mcacheSource=CacheModel::getInstance('source_url'); $mcacheLevel=CacheModel::getInstance('level_url'); $mcacheCont=CacheModel::getInstance('cont_url'); foreach ($this->cont_urls_list as $cont_key=>$cont_urls){ $source_type=0; if('_source_is_url_'==$cont_key){ $source_type=0; }elseif(strpos($cont_key,'level_')===0){ $source_type=2; }else{ $source_type=1; } if($source_type==2){ if(array_key_exists($cont_key,$this->used_level_urls)){ continue; } }else{ if(array_key_exists($cont_key,$this->used_source_urls)){ continue; } } $finished_cont=true; $cur_c_i=0; foreach ($cont_urls as $cont_url){ $cur_c_i+=1; $md5_cont_url=md5($cont_url); if(array_key_exists($md5_cont_url,$this->used_cont_urls)){ continue; } if($this->config['url_repeat']||$mcollected->getCountByUrl($cont_url)<=0){ if(!empty($this->collected_field_list)){ if($this->set_html_interval()===true){ if(!$this->config['url_repeat']&&$mcollected->getCountByUrl($cont_url)>0){ $this->used_cont_urls[$md5_cont_url]=1; continue; } } } if(input('?backstage')){ $backstageDate=CacheModel::getInstance('backstage_task')->db()->field('dateline')->where('cname',$this->collector['task_id'])->find(); if(empty($backstageDate)||$GLOBALS['backstage_task_runtime']<$backstageDate['dateline']){ unset($GLOBALS['backstage_task_ids'][$this->collector['task_id']]); exit('终止进程'); } } if($mcacheCont->getCount($md5_cont_url)>0){ $this->used_cont_urls[$md5_cont_url]=1; continue; } $mcacheCont->setCache($md5_cont_url, 1); $this->echo_msg($echo_str."采集内容页：{$cont_url}",'black'); $field_vals_list=$this->getFields($cont_url); $is_loop=empty($this->first_loop_field)?false:true; if(!empty($field_vals_list)){ $is_real_time=false; if(!empty($GLOBALS['config']['caiji']['real_time'])&&!empty($GLOBALS['real_time_release'])){ $is_real_time=true; } if(!$is_loop){ $field_vals_list=array($field_vals_list); }else{ $loop_cont_urls=array(); foreach ($field_vals_list as $k=>$field_vals){ $loop_cont_urls[$k]=$cont_url.'#'.md5(serialize($field_vals)); } if(!empty($loop_cont_urls)){ $loop_exists_urls=$mcollected->getUrlByUrl($loop_cont_urls); if(!empty($loop_exists_urls)){ $loop_exists_urls=array_flip($loop_exists_urls); foreach ($loop_cont_urls as $k=>$loop_cont_url){ if(isset($loop_exists_urls[$loop_cont_url])){ unset($field_vals_list[$k]); } } $this->echo_msg($echo_str.'已过滤'.count($loop_exists_urls).'条重复数据','black'); } } if(isset($this->exclude_cont_urls[$md5_cont_url])){ $excludeNum=0; foreach($this->exclude_cont_urls[$md5_cont_url] as $k=>$v){ $excludeNum+=count($v); } $this->echo_msg($echo_str.'通过数据处理排除了'.$excludeNum.'条数据','black'); } $field_vals_list=array_values($field_vals_list); } foreach ($field_vals_list as $field_vals){ $collected_error=''; $collected_data=array('url'=>$cont_url,'fields'=>$field_vals); if($is_loop){ $collected_data['url'].='#'.md5(serialize($field_vals)); }else{ if(isset($this->exclude_cont_urls[$md5_cont_url])){ $collected_error=reset($this->exclude_cont_urls[$md5_cont_url]); $collected_error=$this->exclude_url_msg($collected_error); } } if(empty($collected_error)){ if(!empty($this->config['field_title'])){ $collected_data['title']=$field_vals[$this->config['field_title']]['value']; } if(!empty($collected_data['title'])){ if($mcollected->getCountByTitle($collected_data['title'])>0){ $collected_error='标题重复：'.mb_substr($collected_data['title'],0,300,'utf-8'); } } } if(empty($collected_error)){ if($is_real_time){ $GLOBALS['real_time_release']->export(array($collected_data)); unset($collected_data['fields']); unset($collected_data['title']); } $this->collected_field_list[]=$collected_data; }else{ if(!$this->config['url_repeat']){ controller('ReleaseBase','event')->record_collected($collected_data['url'], array('id'=>0,'error'=>$collected_error),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module']) ); }else{ $this->echo_msg($collected_error); } } } } if($is_loop){ controller('ReleaseBase','event')->record_collected( $cont_url,array('id'=>1,'target'=>'','desc'=>'循环入库'),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module']),null,false ); } }else{ $this->echo_msg('已采集过该网址','black'); } $this->used_cont_urls[$md5_cont_url]=1; if($this->collect_num>0){ if(count($this->collected_field_list)>=$this->collect_num){ if($cur_c_isetCache(md5($cont_key),$cont_key); }elseif($source_type==2){ $mcacheLevel->setCache(md5($cont_key),$cont_key); } if($source_type==2){ $this->used_level_urls[$cont_key]=1; }else{ $this->used_source_urls[$cont_key]=1; } } if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){ break; } } } } ?>