You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
91 lines
3.2 KiB
91 lines
3.2 KiB
2 months ago
|
<?php
|
||
|
|
ini_set('display_errors', 1); // 显示错误
|
||
|
|
ini_set('display_startup_errors', 1); // 显示 PHP 启动时的错误
|
||
|
|
error_reporting(E_ERROR); // 记录所有级别的错误
|
||
|
|
|
||
|
|
include_once dirname(dirname(__FILE__)).'/base/dealBase.php';
|
||
|
|
|
||
|
|
define('DEAL_FLAG', $argv[0]);
|
||
|
|
define('PROC_CODE', $argv[1]);
|
||
|
|
|
||
|
|
class getWeiboPic extends dealBase {
|
||
|
|
|
||
|
|
private $task_key = _RQ_SPIDER_WEIBO_DATA;
|
||
|
|
|
||
|
|
public function setPara() {
|
||
|
|
$this->spiderdaemon = SPIDER_WEIBO_DATA;
|
||
|
|
$this->processnum = $GLOBALS['DAEMON_NUMLIMIT'][SPIDER_WEIBO_DATA];
|
||
|
|
}
|
||
|
|
|
||
|
|
public function deal() {
|
||
|
|
$obj = new mSpider();
|
||
|
|
$rdobj = $obj->initRedis();
|
||
|
|
|
||
|
|
// $ipinfo = $rdobj->get(_RQ_SPIDER_USE_IP);
|
||
|
|
// $ipdata = json_decode($ipinfo, true);
|
||
|
|
// if (!$ipinfo && PROC_CODE == 0 && $rdobj->llen($this->task_key) + 0 >= 0) {
|
||
|
|
// $ipdata = $obj->getZmhttpIp(_RQ_SPIDER_USE_IP);
|
||
|
|
// }
|
||
|
|
// if (empty($ipdata)) {
|
||
|
|
// sleep(2);
|
||
|
|
// return false;
|
||
|
|
// }
|
||
|
|
//
|
||
|
|
// $obj->proxy_ip = $ipdata['ip'];
|
||
|
|
// $obj->proxy_port = $ipdata['port'];
|
||
|
|
|
||
|
|
$rq_data = $rdobj->rpop($this->task_key);
|
||
|
|
if (empty($rq_data)) exit();
|
||
|
|
$ipdata['ip'] = '127.0.0.1';
|
||
|
|
$ipdata['port'] = '3306';
|
||
|
|
|
||
|
|
$baselog = $rq_data . "|" . $ipdata['ip'] . "|" . $ipdata['port'] . "|" . PROC_CODE . "|";
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_progress.log', $baselog);
|
||
|
|
|
||
|
|
$mweibo = new mWeibo();
|
||
|
|
$data = $mweibo->getWeiboById($rq_data);
|
||
|
|
|
||
|
|
$is_error = false;
|
||
|
|
|
||
|
|
$pic_ids = json_decode($data['pic_ids'], true);
|
||
|
|
foreach ($pic_ids as $pic_id) {
|
||
|
|
$aliobj = new mAliyunOSS();
|
||
|
|
|
||
|
|
$savepath = $mweibo->getWeiboDataPicSavePath($pic_id, $data['created_at']);
|
||
|
|
$is_exist = $aliobj->isExist(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath);
|
||
|
|
if (!$is_exist) {
|
||
|
|
$params = array();
|
||
|
|
$headers = array(
|
||
|
|
"Referer: https://weibo.com/"
|
||
|
|
);
|
||
|
|
$imgData = $obj->getRequest(sprintf(WEIBO_IMG_URL, $pic_id), $params, $headers);
|
||
|
|
if(!$imgData){
|
||
|
|
$rdobj->lpush($this->task_key, $rq_data);
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_get_error.log', $baselog.$pic_id.'|'.$obj->getError());
|
||
|
|
$is_error = true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
$res = $aliobj->simpleUploadStr(ALIYUN_OSS_BUCKET_WEIBO_DATA, $savepath, $imgData);
|
||
|
|
if(!$res) {
|
||
|
|
$rdobj->lpush($this->task_key, $rq_data);
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_upload_error.log', $baselog.$pic_id.'|'.$aliobj->getError());
|
||
|
|
$is_error = true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_upload_success.log', $baselog.$pic_id);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if($is_error){
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_error.log', $baselog);
|
||
|
|
}
|
||
|
|
|
||
|
|
$obj->writeLog('spider', $this->spiderdaemon . '_success.log', $baselog);
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
new getWeiboPic();
|