knowledge/queue/deal/spider_weibo_comment.php

<?php
include_once dirname(dirname(__FILE__)).'/base/dealBase.php';

define('PROC_CODE', $argv[2]);

class spiderWeiboComment extends dealBase {

    private $task_key = _RQ_SPIDER_WEIBO_COMMENT;

    public function setPara() {
        $this->processnum = $GLOBALS['DAEMON_NUMLIMIT'][SPIDER_WEIBO_COMMENT];
        $this->is_while = true;
    }

    public function deal() {
        $obj = new mSpider();
        $rdobj = $obj->initRedis();

        $cookie_json = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE2);
        $baselog = PROC_CODE . "|";
        if(empty($cookie_json)) {
            if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, 'cookie为空:'.$baselog);
            exit;
        }

        $cookie_data = json_decode($cookie_json, true);
        $cookie = $cookie_data[PROC_CODE];
        if(empty($cookie)) {
            if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '未设置该进程cookie:'.$baselog.'|'.$cookie_json);
            exit;
        }

        $ipinfo = $rdobj->get(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
        $ipdata = json_decode($ipinfo, true);
        if (!$ipinfo && PROC_CODE == 0 && ($rdobj->llen($this->task_key) + 0 > 0 || !$this->is_while)) $ipdata = $obj->getZmhttpIp(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
        if (empty($ipdata)) {
            sleep(2);
            return false;
        }

        $obj->proxy_ip = $ipdata['ip'];
        $obj->proxy_port = $ipdata['port'];

        $rq_data = $rdobj->rpop($this->task_key);
        if(empty($rq_data)) exit();

        $rq_data = json_decode($rq_data, true);

        $weibo_id = $rq_data['weibo_id']+0;
        $max_id = $rq_data['max_id']+0;
        if($weibo_id+0 <= 0 || $max_id < 0) {
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, 'queue:数据格式不正确:'.json_decode( $rq_data));
            exit;
        }

        $baselog = $weibo_id . "|" .$max_id . "|" . $ipdata['ip'] . "|" . $ipdata['port'] . "|" . PROC_CODE . "|";

        $wobj = new mWeibo();
        $weibo_info = $wobj->getWeiboById($weibo_id);
        if(empty($weibo_info)) {
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '微博信息不存在:'.$baselog);
            exit;
        }

        $res = $obj->spiderComments($cookie, $weibo_info['uid'], $weibo_info['wid'], $weibo_info['mblogid'], $max_id);

        $rdobj = $obj->initRedis();
        if (!$res) {
            $error = $obj->getError();
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '队列评论抓取失败，重试中:'.$error.'|'.$baselog);

            if($error == 'cookie失效') {
                $rdobj->lpush($this->task_key, json_encode($rq_data));
                unset($cookie_data[PROC_CODE]);
                $rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data));
                if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);

                $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error);
                sleep(10);
                // $rdobj->del($this->task_key);
                // $rdobj->del(_RS_SPIDER_WEIBO_COMMENT);
                exit;
            }

            $rdobj->lpush($this->task_key, json_encode($rq_data));
            if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP);
            sleep(30);

            return false;
        }

        $cobj = new mWeiboComments();
        $res = $cobj->addComment($weibo_id, $res);
        if(!$res) {
            $error = $cobj->getError().'|评论json数据保存到数据库失败~';
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, $baselog.'|'.$error);
            $wobj->updateWeibo($weibo_id, array('spider_comment_status' => SPIDER_COMMENT_STATUS_ERROR));
            return false;
        }else{
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '微博评论分页抓取成功:'.json_encode($res).'|'.$baselog);
        }

        if(!$res['is_load_all']) {
            $rdobj->lpush($this->task_key, json_encode($res));
            return true;
        }

        $res = $wobj->updateWeibo($weibo_id, array('spider_comment_status' => SPIDER_COMMENT_STATUS_SUCCESS));
        if(!$res) {
            $error = $wobj->getError();
            $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '更新微博状态失败:'.$error.'|'.$baselog);
            sleep(2);
            return false;
        }

        $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取成功:'.$baselog);
        $rdobj->sRem(_RS_SPIDER_WEIBO_COMMENT, $weibo_id);

        sleep(4);

        return true;
    }
}

new spiderWeiboComment();