Browse Source

日志中增加redis设置结果日志

pull/73/head
longchao 3 days ago
parent
commit
eb585c0ff3
  1. 16
      queue/deal/spider_weibo_comment.php

16
queue/deal/spider_weibo_comment.php

@ -64,11 +64,13 @@ class spiderWeiboComment extends dealBase {
$res = $obj->spiderComments($cookie, $weibo_info['uid'], $weibo_info['wid'], $weibo_info['mblogid'], $max_id); $res = $obj->spiderComments($cookie, $weibo_info['uid'], $weibo_info['wid'], $weibo_info['mblogid'], $max_id);
$rdobj = $obj->initRedis();
if (!$res) { if (!$res) {
$error = $obj->getError(); $error = $obj->getError();
if($error == 'cookie失效') { if($error == 'cookie失效') {
sleep(10);
$rdobj = $obj->initRedis();
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data)); $rds = $rdobj->lpush($this->task_key, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
@ -78,18 +80,15 @@ class spiderWeiboComment extends dealBase {
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
$obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error); $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error);
sleep(10);
// $rdobj->del($this->task_key);
// $rdobj->del(_RS_SPIDER_WEIBO_COMMENT);
exit; exit;
} }
sleep(30);
$rdobj = $obj->initRedis();
$rds = $rdobj->lpush($this->task_key, json_encode($rq_data)); $rds = $rdobj->lpush($this->task_key, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败,重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
sleep(30);
return false; return false;
} }
@ -106,6 +105,8 @@ class spiderWeiboComment extends dealBase {
} }
if(!$res['is_load_all']) { if(!$res['is_load_all']) {
sleep(4);
$rdobj = $obj->initRedis();
$rds = $rdobj->lpush($this->task_key, json_encode($res)); $rds = $rdobj->lpush($this->task_key, json_encode($res));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds);
@ -120,6 +121,7 @@ class spiderWeiboComment extends dealBase {
return false; return false;
} }
$rdobj = $obj->initRedis();
$rds = $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id); $rds = $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog.'|redis:'.$rds);

Loading…
Cancel
Save