修复微博未采集完所有评论就停止问题

model/mSpider.php

@@ -347,7 +347,6 @@ class mSpider extends mBase {
         $url = "https://weibo.com/ajax/statuses/buildComments?flow=1&is_reload=1&id={$wid}&is_show_bulletin=2&is_mix=0&count=10&uid={$uid}&fetch_level=0&locale=zh-CN";
         if($max_id > 0) $url = "https://weibo.com/ajax/statuses/buildComments?flow=1&is_reload=1&id={$wid}&is_show_bulletin=2&is_mix=0&max_id={$max_id}&count=10&uid={$uid}&fetch_level=0&locale=zh-CN";
 
-
         $headers = array(
             "Cookie: SUB={$cookie}",
             "x-requested-with: XMLHttpRequest",
@@ -366,7 +365,7 @@ class mSpider extends mBase {
     }
 
     public function requestComment($uid, $url, $headers) {
-        $res = $this->getRequest($url, $headers, 30, true);
+        $res = $this->getRequest($url, $headers, 10, true);
         if (strpos($res, '登录 - 微博')) {
             $this->setError('cookie失效');
             return false;

queue/deal/spider_weibo_comment.php

@@ -13,7 +13,6 @@ class spiderWeiboComment extends dealBase {
     }
 
     public function deal() {
-        exit;
         $obj = new mSpider();
         $rdobj = $obj->initRedis();
 
@@ -69,18 +68,18 @@ class spiderWeiboComment extends dealBase {
             $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog);
 
             if($error == 'cookie失效') {
-                $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error);
+                $rdobj->rpush($this->task_key, json_encode($rq_data));
-
                 unset($cookie_data[PROC_CODE]);
                 $rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data));
-                $rdobj->lpush($this->task_key, json_encode($rq_data));
                 if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
+
+                $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error);
                 // $rdobj->del($this->task_key);
                 // $rdobj->del(_RS_SPIDER_WEIBO_COMMENT);
                 exit;
             }
 
-            $rdobj->lpush($this->task_key, json_encode($rq_data));
+            $rdobj->rpush($this->task_key, json_encode($rq_data));
             if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
             sleep(30);
 
@@ -97,7 +96,7 @@ class spiderWeiboComment extends dealBase {
         }
 
         if(!$res['is_load_all']) {
-            $rdobj->lpush($this->task_key, json_encode($res));
+            $rdobj->rpush($this->task_key, json_encode($res));
             return true;
         }