You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
79 lines
2.5 KiB
79 lines
2.5 KiB
<?php
|
|
include_once dirname(dirname(__FILE__)).'/base/dealBase.php';
|
|
|
|
define('DEAL_FLAG', $argv[0]);
|
|
define('PROC_CODE', $argv[1]);
|
|
|
|
class getWeiboBehavior extends dealBase {
|
|
|
|
private $task_key = _RQ_SPIDER_WEIBO_BEHAVIOR;
|
|
|
|
public function setPara() {
|
|
$this->spiderdaemon = SPIDER_WEIBO_BEHAVIOR;
|
|
$this->processnum = $GLOBALS['DAEMON_NUMLIMIT'][SPIDER_WEIBO_BEHAVIOR];
|
|
}
|
|
|
|
public function deal() {
|
|
$this->is_change_ip = false;
|
|
|
|
$obj = new mSpider();
|
|
$rdobj = $obj->initRedis();
|
|
|
|
$ipinfo = $rdobj->get(_RC_SPIDER_USE_IP);
|
|
$ipdata = json_decode($ipinfo, true);
|
|
if (!$ipinfo && PROC_CODE == 0 && $rdobj->llen($this->task_key) + 0 >= 0) {
|
|
$ipdata = $obj->getZmhttpIp(_RC_SPIDER_USE_IP);
|
|
}
|
|
if (empty($ipdata)) {
|
|
sleep(2);
|
|
return false;
|
|
}
|
|
|
|
$obj->proxy_ip = $ipdata['ip'];
|
|
$obj->proxy_port = $ipdata['port'];
|
|
|
|
$rq_data = $rdobj->rpop($this->task_key);
|
|
if (empty($rq_data)) {
|
|
sleep(60);
|
|
return false;
|
|
}
|
|
|
|
$baselog = $rq_data . "|" . $ipdata['ip'] . "|" . $ipdata['port'] . "|" . PROC_CODE . "|";
|
|
|
|
//数据格式不正确
|
|
$data = json_decode($rq_data, true);
|
|
if(empty($data['type']) || empty($data['uid']) || empty($data['page']) || empty($data['date'])) return false;
|
|
|
|
//队列数据已过期
|
|
$date = date("Y-m-d");
|
|
if($data['date'] != $date) return false;
|
|
|
|
$cookie = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE);
|
|
if(empty($cookie)) $cookie = WEIBO_LOGIN_COOKIE;
|
|
|
|
if($data['type'] == 1){
|
|
//原创数据
|
|
$res = $obj->spiderBehaviorOriginalData($cookie, $data['uid'], $data['page']);
|
|
}else{
|
|
//转发数据
|
|
$res = $obj->spiderBehaviorForwardData($cookie, $data['uid'], $data['page']);
|
|
}
|
|
|
|
if (!$res) {
|
|
$error = $obj->getError();
|
|
if($error == 'cookie失效') $obj->sendMail(array('1026652509@qq.com'), date('Y年m月d日', time()) . '-微博数据抓取', $data['info']);
|
|
if ($obj->is_change_ip) {
|
|
// 0进程监控更换ip
|
|
if (PROC_CODE == 0) $rdobj->del(_RC_SPIDER_USE_IP);
|
|
}
|
|
$obj->writeLog('spider', $this->spiderdaemon.'_fail.log', $baselog.$error);
|
|
$rdobj->lpush($this->task_key, $rq_data);
|
|
return false;
|
|
}
|
|
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_success.log', $baselog);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
new getWeiboBehavior();
|
|
|