es录入

queue/deal/get_weibo_data.php

@@ -1,90 +0,0 @@
-<?php
-ini_set('display_errors', 1);      // 显示错误
-ini_set('display_startup_errors', 1); // 显示 PHP 启动时的错误
-error_reporting(E_ERROR);            // 记录所有级别的错误
-
-include_once dirname(dirname(__FILE__)).'/base/dealBase.php';
-
-define('DEAL_FLAG', $argv[0]);
-define('PROC_CODE', $argv[1]);
-
-class getWeiboData extends dealBase {
-
-    private $task_key = _RQ_SPIDER_WEIBO_DATA;
-
-    public function setPara() {
-        $this->spiderdaemon = SPIDER_WEIBO_DATA;
-        $this->processnum = $GLOBALS['DAEMON_NUMLIMIT'][SPIDER_WEIBO_DATA];
-    }
-
-    public function deal() {
-        $obj = new mSpider();
-        $rdobj = $obj->initRedis();
-
-//        $ipinfo = $rdobj->get(_RQ_SPIDER_USE_IP);
-//        $ipdata = json_decode($ipinfo, true);
-//        if (!$ipinfo && PROC_CODE == 0 && $rdobj->llen($this->task_key) + 0 >= 0) {
-//            $ipdata = $obj->getZmhttpIp(_RQ_SPIDER_USE_IP);
-//        }
-//        if (empty($ipdata)) {
-//            sleep(2);
-//            return false;
-//        }
-//
-//        $obj->proxy_ip = $ipdata['ip'];
-//        $obj->proxy_port = $ipdata['port'];
-
-        $rq_data = $rdobj->rpop($this->task_key);
-        if (empty($rq_data)) exit();
-        $ipdata['ip'] = '127.0.0.1';
-        $ipdata['port'] = '3306';
-
-        $baselog = $rq_data . "|" . $ipdata['ip'] . "|" . $ipdata['port'] . "|" . PROC_CODE . "|";
-        $obj->writeLog('spider', $this->spiderdaemon . '_progress.log', $baselog);
-
-        $mweibo = new mWeibo();
-        $data = $mweibo->getWeiboById($rq_data);
-
-        $is_error = false;
-
-        $pic_ids = json_decode($data['pic_ids'], true);
-        foreach ($pic_ids as $pic_id) {
-            $aliobj = new mAliyunOSS();
-
-            $savepath = $mweibo->getWeiboDataPicSavePath($pic_id, $data['created_at']);
-            $is_exist = $aliobj->isExist(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath);
-            if (!$is_exist) {
-                $params = array();
-                $headers = array(
-                    "Referer: https://weibo.com/"
-                );
-                $imgData = $obj->getRequest(sprintf(WEIBO_IMG_URL, $pic_id), $params, $headers);
-                if(!$imgData){
-                    $rdobj->lpush($this->task_key, $rq_data);
-                    $obj->writeLog('spider', $this->spiderdaemon . '_get_error.log', $baselog.$pic_id.'|'.$obj->getError());
-                    $is_error = true;
-                    break;
-                }
-
-                $res = $aliobj->simpleUploadStr(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath, $imgData);
-                if(!$res) {
-                    $rdobj->lpush($this->task_key, $rq_data);
-                    $obj->writeLog('spider', $this->spiderdaemon . '_upload_error.log', $baselog.$pic_id.'|'.$aliobj->getError());
-                    $is_error = true;
-                    break;
-                }
-
-                $obj->writeLog('spider', $this->spiderdaemon . '_upload_success.log', $baselog.$pic_id);
-            }
-        }
-
-        if($is_error){
-            $obj->writeLog('spider', $this->spiderdaemon . '_error.log', $baselog);
-        }
-
-        $obj->writeLog('spider', $this->spiderdaemon . '_success.log', $baselog);
-        return true;
-    }
-}
-
-new getWeiboData();