Browse Source

Merge pull request '日志中增加redis设置结果日志' (#72) from xmz_weibo_demo into master

Reviewed-on: #72
pull/77/head
longchao 2 days ago
parent
commit
786003e498
  1. 19
      queue/deal/spider_weibo_comment.php

19
queue/deal/spider_weibo_comment.php

@ -67,10 +67,12 @@ class spiderWeiboComment extends dealBase {
$rdobj = $obj->initRedis();
if (!$res) {
$error = $obj->getError();
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog);
if($error == 'cookie失效') {
$rdobj->lpush($this->task_key, json_encode($rq_data));
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
unset($cookie_data[PROC_CODE]);
$rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data));
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
@ -82,7 +84,10 @@ class spiderWeiboComment extends dealBase {
exit;
}
$rdobj->lpush($this->task_key, json_encode($rq_data));
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
sleep(30);
@ -101,7 +106,9 @@ class spiderWeiboComment extends dealBase {
}
if(!$res['is_load_all']) {
$rdobj->lpush($this->task_key, json_encode($res));
$rds = $rdobj->lpush($this->task_key, json_encode($res));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds);
return true;
}
@ -113,8 +120,8 @@ class spiderWeiboComment extends dealBase {
return false;
}
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog);
$rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);
$rds = $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog.'|redis:'.$rds);
sleep(4);

Loading…
Cancel
Save