You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
95 lines
3.2 KiB
95 lines
3.2 KiB
<?php
|
|
// nohup php /data1/www/zhishiku.kuailelunwen.com/tools/es_setting.php &
|
|
include_once(dirname(dirname(__FILE__))."/vendor/autoload.php");
|
|
include_once(dirname(dirname(__FILE__))."/tools/es.php");
|
|
|
|
function parseDbCnf($dbflag) {
|
|
$configs = parse_ini_file(dirname(dirname(__FILE__)).'/config/database.ini', true);
|
|
$config = $configs[strtolower($dbflag)];
|
|
|
|
$host_port = $config['master'];
|
|
|
|
$hps = explode(',', $host_port);
|
|
$random = rand(0, count($hps)-1);
|
|
$hp = $hps[$random];
|
|
|
|
list($host, $port) = explode(':', $hp);
|
|
|
|
$cnf = array();
|
|
$cnf['host'] = trim($host);
|
|
$cnf['port'] = trim($port);
|
|
$cnf['user'] = trim($config['user']);
|
|
$cnf['pwd'] = trim($config['passwd']);
|
|
$cnf['db'] = trim($config['db']);
|
|
return $cnf;
|
|
}
|
|
|
|
$mysqlconfig = parseDbCnf('simplyphp');
|
|
$servername = $mysqlconfig['host'];
|
|
$username = $mysqlconfig['user'];
|
|
$password = $mysqlconfig['pwd'];
|
|
$dbname = $mysqlconfig['db'];
|
|
|
|
$obj = new \ES('weibo');
|
|
$conn = new mysqli($servername, $username, $password, $dbname);
|
|
if ($conn->connect_error) {
|
|
die("连接失败: " . $conn->connect_error);
|
|
}
|
|
|
|
$conn->set_charset("utf8mb4");
|
|
|
|
$limit = 5000;
|
|
$min_id = 0;
|
|
|
|
$log_path = '/datacenter/zhishiku/es.log';
|
|
$log_path_success = '/datacenter/zhishiku/es_success.log';
|
|
$log_path_err = '/datacenter/zhishiku/es_error.log';
|
|
|
|
for($page=0;;$page++){
|
|
$sql = "SELECT * FROM spider_weibo where uid=2282201403 and id>".$min_id." order by id asc limit ".$limit;
|
|
$result = $conn->query($sql);
|
|
if ($result->num_rows > 0) {
|
|
while($row = $result->fetch_assoc()) {
|
|
$min_id = $row['id'];
|
|
|
|
if(str_replace(" ", "", $row["text"])=="") continue;
|
|
|
|
$data['id'] = $row['id']+0;
|
|
$data['uid'] = $row['uid']+0;
|
|
$data['wid'] = $row['wid']+0;
|
|
$data['content'] = $row['text'];
|
|
|
|
if (!empty($row['created_at'])) {
|
|
$data['created_at'] = date('c', strtotime($row['created_at']));
|
|
} else {
|
|
$data['created_at'] = null;
|
|
}
|
|
$data['comments_count'] = $row['comments_count']+0;
|
|
$data['attitudes_count'] = $row['attitudes_count']+0;
|
|
$data['reposts_count'] = $row['reposts_count']+0;
|
|
$data['status'] = $row['status']+0;
|
|
|
|
$params['index'] = 'weibo';
|
|
$params['type'] = 'doc';
|
|
$params['id'] = $data['id'];
|
|
$params['body'] = $data;
|
|
try {
|
|
$resc = $obj->add_doc($params);
|
|
if($resc['result'] != 'created') {
|
|
error_log('error:'.json_encode($data)."\n", 3, $log_path_err);
|
|
}else{
|
|
error_log($resc['_id']."|{$min_id}\n", 3, $log_path_success);
|
|
}
|
|
} catch (\Exception $th) {
|
|
error_log('excption:'.$th->getMessage().'|'.json_encode($data).'|'.$min_id."\n", 3, $log_path_err);
|
|
}
|
|
}
|
|
error_log((($page*$limit)+$result->num_rows)."|".$page."\n", 3, $log_path);
|
|
if($result->num_rows<$limit) break;
|
|
} else {
|
|
error_log((($page*$limit)+$result->num_rows)."|".$page."\n", 3, $log_path);
|
|
break;
|
|
}
|
|
}
|
|
$conn->close();
|
|
?>
|
|
|