延长抓取时间

model/mSpider.php

@@ -365,7 +365,7 @@ class mSpider extends mBase {
     }
 
     public function requestComment($uid, $url, $headers) {
-        $res = $this->getRequest($url, $headers, 10, true);
+        $res = $this->getRequest($url, $headers, 40, true);
         if (strpos($res, '登录 - 微博')) {
             $this->setError('cookie失效');
             return false;

queue/deal/spider_weibo_comment.php

@@ -68,18 +68,19 @@ class spiderWeiboComment extends dealBase {
             $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog);
 
             if($error == 'cookie失效') {
-                $rdobj->rpush($this->task_key, json_encode($rq_data));
+                $rdobj->lpush($this->task_key, json_encode($rq_data));
                 unset($cookie_data[PROC_CODE]);
                 $rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data));
                 if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
 
                 $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error);
+                sleep(10);
                 // $rdobj->del($this->task_key);
                 // $rdobj->del(_RS_SPIDER_WEIBO_COMMENT);
                 exit;
             }
 
-            $rdobj->rpush($this->task_key, json_encode($rq_data));
+            $rdobj->lpush($this->task_key, json_encode($rq_data));
             if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
             sleep(30);
 
@@ -98,7 +99,7 @@ class spiderWeiboComment extends dealBase {
         }
 
         if(!$res['is_load_all']) {
-            $rdobj->rpush($this->task_key, json_encode($res));
+            $rdobj->lpush($this->task_key, json_encode($res));
             return true;
         }
 
@@ -112,7 +113,7 @@ class spiderWeiboComment extends dealBase {
 
         $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog);
         $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);
-        sleep(2);
+        sleep(4);
 
         return true;
     }