obj = new dWeiboComments(); $this->tbl = 'spider_weibo_comments'; } public function getCommentByWeiboId($weibo_id, $page = 0, $limit = 0, $order = 'id asc') { $limit_info = array(); if($page > 0 && $limit > 0) $limit_info = array(($page-1)*$limit, $limit); return $this->obj->selectAll($this->tbl, array('sql' => '`weibo_id`=?', 'vals' => array($weibo_id)), $order, $limit_info); } public function getCommentByWeiboDataId($weibo_data_id) { return $this->obj->select($this->tbl, array('sql' => '`weibo_data_id`=?', 'vals' => array($weibo_data_id))); } public function isNeedInsertData($weibo_id, $max_weibo_data_id) { $max_weibo_data_id_info = $this->getCommentByWeiboId($max_weibo_data_id, 1, 1); if(empty($max_weibo_data_id_info)) return true; $weibo_info = $this->getCommentByWeiboId($weibo_id, 1, 1); if(empty($weibo_info)) return true; if($max_weibo_data_id <= $weibo_info[0]['weibo_data_id']) return false; return true; } public function addComment($weibo_id, $source_json) { $obj = new mWeibo(); $weibo_info = $obj->getWeiboById($weibo_id); if(empty($weibo_info)) { $this->setError('微博不存在'); $this->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '微博不存在:'.$weibo_id); return false; } $source_data = json_decode($source_json, true); if($source_data['ok'] != 1) { $this->setError('抓取失败'); $this->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '抓取评论失败:'.$source_json); return false; } if(!is_dir(ZHISHIKU_SPIDER_TEMP_PATH)) { mkdir(ZHISHIKU_SPIDER_TEMP_PATH, 0755, true); chown(ZHISHIKU_SPIDER_TEMP_PATH, 'nobody'); chgrp(ZHISHIKU_SPIDER_TEMP_PATH, 'nobody'); } foreach($source_data['data'] as $key=>$comment) { $weibo_data_id = $comment['id']+0; $source_json_save_path = sprintf(ZHISHIKU_SPIDER_COMMENT_PATH, $weibo_info['wid'], $weibo_data_id); $dir = dirname(dirname($source_json_save_path)); if(!is_dir($dir)) { mkdir($dir, 0755, true); chown($dir, 'nobody'); chgrp($dir, 'nobody'); } $dir = dirname($source_json_save_path); if(!is_dir($dir)) { mkdir($dir, 0755, true); chown($dir, 'nobody'); chgrp($dir, 'nobody'); } if(!is_dir($dir)) { $this->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '创建目录失败:'.$source_json_save_path); continue; } if(file_exists($source_json_save_path)) continue; file_put_contents($source_json_save_path, json_encode($comment), LOCK_EX); if(!file_exists($source_json_save_path) || filesize($source_json_save_path) < 10) { $this->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '评论保存到文件失败:'.$source_json_save_path); unset($source_data['data'][$key]); continue; } chmod($source_json_save_path, 0755); chown($source_json_save_path, 'nobody'); chgrp($source_json_save_path, 'nobody'); } if(empty($source_data['data'])) return true; $sqls = array(); foreach($source_data['data'] as $comment) { $content = $comment['text']; $weibo_data_id = $comment['id']+0; $comment_time = date('Y-m-d H:i:s', strtotime($comment['created_at'])); $sqls[] = array( 'sql' => 'insert into '.$this->tbl.' (`weibo_id`, `weibo_data_id`, `content`, `comment_time`) values (?, ?, ?, ?) ON DUPLICATE KEY UPDATE `weibo_data_id`=?', 'vals' => array($weibo_id, $weibo_data_id, $content, $comment_time, $weibo_data_id), ); } $res = $this->obj->execTrans2($sqls); if(!$res) { $this->setError('保存评论失败'); $this->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '保存评论失败:'.json_encode($sqls)); return false; } return true; } }