zhangqi
/
knowledge
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity
Browse Source

日志中增加redis设置结果日志

pull/75/head
longchao 3 days ago
parent
1cc575533d
commit
8b947033dc
1 changed files with 7 additions and 7 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 14
      queue/deal/spider_weibo_comment.php

14
queue/deal/spider_weibo_comment.php
View File

@ -19,20 +19,20 @@ class spiderWeiboComment extends dealBase {
$cookie_json = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE2); $cookie_json = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE2);
$baselog = PROC_CODE . "|"; $baselog = PROC_CODE . "|";
if(empty($cookie_json)) { if(empty($cookie_json)) {
if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, 'cookie为空:'.$baselog); if($rdobj->llen(_RQ_SPIDER_WEIBO_COMMENT) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, 'cookie为空:'.$baselog);
exit; exit;
} }
$cookie_data = json_decode($cookie_json, true); $cookie_data = json_decode($cookie_json, true);
$cookie = $cookie_data[PROC_CODE]; $cookie = $cookie_data[PROC_CODE];
if(empty($cookie)) { if(empty($cookie)) {
if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '未设置该进程cookie:'.$baselog.'|'.$cookie_json); if($rdobj->llen(_RQ_SPIDER_WEIBO_COMMENT) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '未设置该进程cookie:'.$baselog.'|'.$cookie_json);
exit; exit;
} }
$ipinfo = $rdobj->get(_RC_SPIDER_WEIBO_COMMENT_USE_IP); $ipinfo = $rdobj->get(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
$ipdata = json_decode($ipinfo, true); $ipdata = json_decode($ipinfo, true);
if (!$ipinfo && PROC_CODE == 0 && ($rdobj->llen($this->task_key) + 0 > 0 || !$this->is_while)) $ipdata = $obj->getZmhttpIp(_RC_SPIDER_WEIBO_COMMENT_USE_IP); if (!$ipinfo && PROC_CODE == 0 && ($rdobj->llen(_RQ_SPIDER_WEIBO_COMMENT) + 0 > 0 || !$this->is_while)) $ipdata = $obj->getZmhttpIp(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
if (empty($ipdata)) { if (empty($ipdata)) {
sleep(2); sleep(2);
return false; return false;
@ -41,7 +41,7 @@ class spiderWeiboComment extends dealBase {
$obj->proxy_ip = $ipdata['ip']; $obj->proxy_ip = $ipdata['ip'];
$obj->proxy_port = $ipdata['port']; $obj->proxy_port = $ipdata['port'];
$rq_data = $rdobj->rpop($this->task_key); $rq_data = $rdobj->rpop(_RQ_SPIDER_WEIBO_COMMENT);
if(empty($rq_data)) exit(); if(empty($rq_data)) exit();
$rq_data = json_decode($rq_data, true); $rq_data = json_decode($rq_data, true);
@ -71,7 +71,7 @@ class spiderWeiboComment extends dealBase {
if($error == 'cookie失效') { if($error == 'cookie失效') {
sleep(30); sleep(30);
$rdobj = $obj->initRedis(); $rdobj = $obj->initRedis();
$rds = $rdobj->rpush($this->task_key, json_encode($rq_data)); $rds = $rdobj->rpush(_RQ_SPIDER_WEIBO_COMMENT, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
@ -86,7 +86,7 @@ class spiderWeiboComment extends dealBase {
sleep(30); sleep(30);
$rdobj = $obj->initRedis(); $rdobj = $obj->initRedis();
$rds = $rdobj->rpush($this->task_key, json_encode($rq_data)); $rds = $rdobj->rpush(_RQ_SPIDER_WEIBO_COMMENT, json_encode($rq_data));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog.'|redis:'.$rds);
if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
@ -107,7 +107,7 @@ class spiderWeiboComment extends dealBase {
if(!$res['is_load_all']) { if(!$res['is_load_all']) {
sleep(10); sleep(10);
$rdobj = $obj->initRedis(); $rdobj = $obj->initRedis();
$rds = $rdobj->rpush($this->task_key, json_encode($res)); $rds = $rdobj->rpush(_RQ_SPIDER_WEIBO_COMMENT, json_encode($res));
$obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds); $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取下一页:'.$baselog.'|redis:'.$rds);
return true; return true;

Write Preview
Loading…
Cancel
Save