|
|
@ -67,10 +67,12 @@ class spiderWeiboComment extends dealBase { |
|
|
|
$rdobj = $obj->initRedis(); |
|
|
|
if (!$res) { |
|
|
|
$error = $obj->getError(); |
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog); |
|
|
|
|
|
|
|
if($error == 'cookie失效') { |
|
|
|
$rdobj->lpush($this->task_key, json_encode($rq_data)); |
|
|
|
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data)); |
|
|
|
|
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds); |
|
|
|
|
|
|
|
unset($cookie_data[PROC_CODE]); |
|
|
|
$rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data)); |
|
|
|
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); |
|
|
@ -82,7 +84,10 @@ class spiderWeiboComment extends dealBase { |
|
|
|
exit; |
|
|
|
} |
|
|
|
|
|
|
|
$rdobj->lpush($this->task_key, json_encode($rq_data)); |
|
|
|
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data)); |
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds); |
|
|
|
|
|
|
|
|
|
|
|
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); |
|
|
|
sleep(30); |
|
|
|
|
|
|
@ -101,7 +106,9 @@ class spiderWeiboComment extends dealBase { |
|
|
|
} |
|
|
|
|
|
|
|
if(!$res['is_load_all']) { |
|
|
|
$rdobj->lpush($this->task_key, json_encode($res)); |
|
|
|
$rds = $rdobj->lpush($this->task_key, json_encode($res)); |
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds); |
|
|
|
|
|
|
|
return true; |
|
|
|
} |
|
|
|
|
|
|
@ -113,8 +120,8 @@ class spiderWeiboComment extends dealBase { |
|
|
|
return false; |
|
|
|
} |
|
|
|
|
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog); |
|
|
|
$rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id); |
|
|
|
$rds = $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id); |
|
|
|
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog.'|redis:'.$rds); |
|
|
|
|
|
|
|
sleep(4); |
|
|
|
|
|
|
|