Browse Source

Merge pull request '修复抓取所有评论失败问题' (#91) from xmz_weibo_demo into master

Reviewed-on: #91
master
longchao 2 days ago
parent
commit
a3508e7a19
  1. 9
      queue/deal/spider_weibo_comment.php

9
queue/deal/spider_weibo_comment.php

@ -117,18 +117,19 @@ class spiderWeiboComment extends dealBase {
return false; return false;
}else{ }else{
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '微博评论分页抓取成功:'.json_encode($res).'|'.$baselog); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '微博评论分页抓取成功:'.json_encode($res).'|'.$baselog);
}
if(!$res['is_load_all']) {
$rdobj = $obj->initRedis(); $rdobj = $obj->initRedis();
if($res['count'] == 0 && $res['max_id'] > 0) { if($res['count'] == 0 && $res['max_id'] > 0) {
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取所有评论失败:'.$baselog.'|'.$res['max_id']); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取所有评论失败:'.$baselog.'|'.$res['max_id']);
$ures = $wobj->updateWeibo($weibo_id, array('spider_comment_status' => SPIDER_COMMENT_STATUS_ERROR)); $ures = $wobj->updateWeibo($weibo_id, array('spider_comment_status' => SPIDER_COMMENT_STATUS_ERROR));
if($ures) $rdobj->srem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id); if($ures) $rdobj->srem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);
sleep(10);
return false; return false;
} }
sleep(10); }
if(!$res['is_load_all']) {
sleep(10);
$rdobj = $obj->initRedis();
$rds = $rdobj->rpush(_RQ_SPIDER_WEIBO_COMMENT, json_encode($res)); $rds = $rdobj->rpush(_RQ_SPIDER_WEIBO_COMMENT, json_encode($res));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|result:'.json_encode($res).'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|result:'.json_encode($res).'|redis:'.$rds);
return true; return true;

Loading…
Cancel
Save