spiderdaemon = SPIDER_WEIBO_DATA; $this->processnum = $GLOBALS['DAEMON_NUMLIMIT'][SPIDER_WEIBO_DATA]; } public function deal() { $obj = new mSpider(); $rdobj = $obj->initRedis(); // $ipinfo = $rdobj->get(_RQ_SPIDER_USE_IP); // $ipdata = json_decode($ipinfo, true); // if (!$ipinfo && PROC_CODE == 0 && $rdobj->llen($this->task_key) + 0 >= 0) { // $ipdata = $obj->getZmhttpIp(_RQ_SPIDER_USE_IP); // } // if (empty($ipdata)) { // sleep(2); // return false; // } // // $obj->proxy_ip = $ipdata['ip']; // $obj->proxy_port = $ipdata['port']; $rq_data = $rdobj->rpop($this->task_key); if (empty($rq_data)) exit(); $ipdata['ip'] = '127.0.0.1'; $ipdata['port'] = '3306'; $baselog = $rq_data . "|" . $ipdata['ip'] . "|" . $ipdata['port'] . "|" . PROC_CODE . "|"; $obj->writeLog('spider', $this->spiderdaemon . '_progress.log', $baselog); $mweibo = new mWeibo(); $data = $mweibo->getWeiboById($rq_data); $is_error = false; $pic_ids = json_decode($data['pic_ids'], true); foreach ($pic_ids as $pic_id) { $aliobj = new mAliyunOSS(); $savepath = $mweibo->getWeiboDataPicSavePath($pic_id, $data['created_at']); $is_exist = $aliobj->isExist(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath); if (!$is_exist) { $params = array(); $headers = array( "Referer: https://weibo.com/" ); $imgData = $obj->getRequest(sprintf(WEIBO_IMG_URL, $pic_id), $params, $headers); if(!$imgData){ $rdobj->lpush($this->task_key, $rq_data); $obj->writeLog('spider', $this->spiderdaemon . '_get_error.log', $baselog.$pic_id.'|'.$obj->getError()); $is_error = true; break; } $res = $aliobj->simpleUploadStr(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath, $imgData); if(!$res) { $rdobj->lpush($this->task_key, $rq_data); $obj->writeLog('spider', $this->spiderdaemon . '_upload_error.log', $baselog.$pic_id.'|'.$aliobj->getError()); $is_error = true; break; } $obj->writeLog('spider', $this->spiderdaemon . '_upload_success.log', $baselog.$pic_id); } } if($is_error){ $obj->writeLog('spider', $this->spiderdaemon . '_error.log', $baselog); } $obj->writeLog('spider', $this->spiderdaemon . '_success.log', $baselog); return true; } } new getWeiboPic();