diff --git a/config/define.php b/config/define.php index c2deb26..d2d421a 100644 --- a/config/define.php +++ b/config/define.php @@ -93,4 +93,14 @@ define('_RS_SPIDER_WEIBO_COMMENT', 'rs_spider_weibo_comment'); define('_RC_WEIBO_LOGIN_COOKIE2', 'rc_weibo_login_cookie2'); - define('WEIBO_LOGIN_COOKIE2', '_2A25FdJmRDeRhGeFG71sY9CbOyDuIHXVmC5NZrDV8PUNbmtAYLUj1kW9NeUknrDj_DFv7tUuAQqx1kWSg_y3B4Qe3'); \ No newline at end of file + define('WEIBO_LOGIN_COOKIE2', '_2A25FdJmRDeRhGeFG71sY9CbOyDuIHXVmC5NZrDV8PUNbmtAYLUj1kW9NeUknrDj_DFv7tUuAQqx1kWSg_y3B4Qe3'); + + //该条评论是否能被搜索 -1=未处理 0=否 1=是 + define('SPIDER_COMMENT_SEARCH_STATUS_WAIT', -1); + define('SPIDER_COMMENT_SEARCH_STATUS_NO', 0); + define('SPIDER_COMMENT_SEARCH_STATUS_YES', 1); + $GLOBALS['SPIDER_COMMENT_SEARCH_STATUS_LIST'] = array( + SPIDER_COMMENT_SEARCH_STATUS_WAIT => '待筛选', + SPIDER_COMMENT_SEARCH_STATUS_NO => '已剔除', + SPIDER_COMMENT_SEARCH_STATUS_YES => '筛选通过', + ); diff --git a/control/index.php b/control/index.php index 20027b9..e8fc1cd 100644 --- a/control/index.php +++ b/control/index.php @@ -95,6 +95,7 @@ class index extends publicBase { $this->ajax_json(true, '设置成功'); } + public function interlocution() { } diff --git a/control/weibo.php b/control/weibo.php index 5bd6a6e..92467e4 100644 --- a/control/weibo.php +++ b/control/weibo.php @@ -9,13 +9,63 @@ class weibo extends publicBase { public function home() {} + public function comments() {} + + public function comment_detail() { + $id = $this->get('id') + 0; + + $obj = new mWeibo(); + $weibo = $obj->getWeiboDetailById($id); + if (empty($weibo)) $this->show_message('微博不存在', 'goback'); + + $this->view['data'] = $weibo; + + $cur_page = $this->get('page')+0 > 0 ? $this->get('page')+0 : 1; + $page_size = $this->get('size')+0 > 0 ? $this->get('size')+0 : 20; + + $cond = array(); + $is_search = $this->get('search_type') + 0; + if(isset($_GET['search_type']) && $is_search >= 0) $cond['is_search'] = $is_search; + + $mobj = new mWeiboComments(); + + $total = $mobj->getCommentCountByWeiboId($id, $cond); + $this->view['last_page'] = ceil($total / $page_size); + + $cur_page = $cur_page > $this->view['last_page'] ? $this->view['last_page'] : $cur_page; + $this->view['total'] = $total; + $this->view['per_page'] = $page_size; + $this->view['cur_page'] = $cur_page; + + $comment_list = $mobj->getCommentByWeiboId($id, $cur_page, $page_size, 'id asc', $cond); + if(!empty($comment_list)) { + foreach ($comment_list as $k => $v) { + $comment_save_path = sprintf(ZHISHIKU_SPIDER_COMMENT_PATH, $weibo['wid'], $v['weibo_data_id']); + if(!file_exists($comment_save_path)) $this->show_message('评论数据文件不存在', 'goback'); + $cdata = json_decode(file_get_contents($comment_save_path), true); + $comment_list[$k]['comment_time'] = date('Y-m-d H:i:s', strtotime($cdata['created_at'])); + $comment_list[$k]['source'] = $cdata['source']; + $comment_list[$k]['screen_name'] = $cdata['user']['screen_name']; + } + + $this->view['comment_list'] = $comment_list; + $this->view['search_status_list'] = $GLOBALS['SPIDER_COMMENT_SEARCH_STATUS_LIST']; + } + } + public function ajax_weibo_list() { $status = $this->post('status') + 0; + $spider_comment_status = $this->post('spider_comment_status') + 0; $cur_page = $this->post('currentPage') ? $this->post('currentPage') : 1; $page_size = $this->post('pageSize') ? $this->post('pageSize') : 20; $condition = array(); - $condition['status'] = $status; + if ($spider_comment_status) { + $condition['spider_comment_status'] = $spider_comment_status; + $condition['uid'] = WEIBO_USER_ZHANG; + }else{ + $condition['status'] = $status; + } $obj = new mWeibo(); $total = $obj->getWeiboTotal($condition); diff --git a/model/mWeiboComments.php b/model/mWeiboComments.php index 211319f..18c9310 100644 --- a/model/mWeiboComments.php +++ b/model/mWeiboComments.php @@ -14,10 +14,40 @@ class mWeiboComments extends mBase { $this->tbl = 'spider_weibo_comments'; } - public function getCommentByWeiboId($weibo_id, $page = 0, $limit = 0, $order = 'id asc') { + public function getCommentByWeiboId($weibo_id, $page = 0, $limit = 0, $order = 'id asc', $condition = array()) { + $where = "1=1 "; + if (!empty($condition)) { + foreach ($condition as $key => $val) { + if (is_array($val)) { + $val = implode(',', $val); + $where .= " and {$key} in ({$val})"; + } else { + $where .= " and {$key}={$val}"; + } + } + } + $where.= " and `weibo_id`={$weibo_id}"; + $limit_info = array(); if($page > 0 && $limit > 0) $limit_info = array(($page-1)*$limit, $limit); - return $this->obj->selectAll($this->tbl, array('sql' => '`weibo_id`=?', 'vals' => array($weibo_id)), $order, $limit_info); + return $this->obj->selectAll($this->tbl, array('sql' => $where, 'vals' => array()), $order, $limit_info); + } + + public function getCommentCountByWeiboId($weibo_id, $condition = array()) { + $where = "1=1 "; + if (!empty($condition)) { + foreach ($condition as $key => $val) { + if (is_array($val)) { + $val = implode(',', $val); + $where .= " and {$key} in ({$val})"; + } else { + $where .= " and {$key}={$val}"; + } + } + } + $where.= " and `weibo_id`={$weibo_id}"; + + return $this->obj->count($this->tbl, array('sql' => $where, 'vals' => array())); } public function getCommentByWeiboDataId($weibo_data_id) { diff --git a/queue/crontab/add_spider_weibo_comment_task.php b/queue/crontab/add_spider_weibo_comment_task.php index 0537787..b070dbf 100644 --- a/queue/crontab/add_spider_weibo_comment_task.php +++ b/queue/crontab/add_spider_weibo_comment_task.php @@ -15,7 +15,7 @@ class addSpiderWeiboCommentTask { $obj = new mWeibo(); $rdobj = $obj->initRedis(); - if($rdobj->llen(_RQ_SPIDER_WEIBO_COMMENT)+0 > 50) return true; + if($rdobj->llen(_RQ_SPIDER_WEIBO_COMMENT)+0 > 0) return true; for($page_num=1;$page_num<=1;$page_num++) { $weibo_data = $obj->getWeiboList($condition, $page_num, $page_size); diff --git a/queue/deal/spider_weibo_comment.php b/queue/deal/spider_weibo_comment.php index d276aaa..87016ad 100644 --- a/queue/deal/spider_weibo_comment.php +++ b/queue/deal/spider_weibo_comment.php @@ -16,6 +16,20 @@ class spiderWeiboComment extends dealBase { $obj = new mSpider(); $rdobj = $obj->initRedis(); + $cookie_json = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE2); + $baselog = PROC_CODE . "|"; + if(empty($cookie_json)) { + if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, 'cookie为空:'.$baselog); + exit; + } + + $cookie_data = json_decode($cookie_json, true); + $cookie = $cookie_data[PROC_CODE]; + if(empty($cookie)) { + if($rdobj->llen($this->task_key) + 0 > 0) $obj->writeLog(ZHISHIKU_SPIDER_LOG, ZHISHIKU_SPIDER_COMMENT, '未设置该进程cookie:'.$baselog.'|'.$cookie_json); + exit; + } + $ipinfo = $rdobj->get(_RC_SPIDER_WEIBO_COMMENT_USE_IP); $ipdata = json_decode($ipinfo, true); if (!$ipinfo && PROC_CODE == 0 && ($rdobj->llen($this->task_key) + 0 > 0 || !$this->is_while)) $ipdata = $obj->getZmhttpIp(_RC_SPIDER_WEIBO_COMMENT_USE_IP); @@ -48,9 +62,6 @@ class spiderWeiboComment extends dealBase { exit; } - $cookie = $rdobj->get(_RC_WEIBO_LOGIN_COOKIE2); - if(empty($cookie)) $cookie = WEIBO_LOGIN_COOKIE2; - $res = $obj->spiderComments($cookie, $weibo_info['uid'], $weibo_info['wid'], $weibo_info['mblogid'], $max_id); if (!$res) { $error = $obj->getError(); @@ -58,8 +69,13 @@ class spiderWeiboComment extends dealBase { if($error == 'cookie失效') { $obj->sendMail(array('1432334894@qq.com'), date('Y年m月d日', time()) . '-微博评论数据抓取', $error); - $rdobj->del($this->task_key); - $rdobj->del(_RS_SPIDER_WEIBO_COMMENT); + + unset($cookie_data[PROC_CODE]); + $rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookie_data)); + $rdobj->lpush($this->task_key, json_encode($rq_data)); + if ($obj->is_change_ip && PROC_CODE == 0) $rdobj->del(_RC_SPIDER_WEIBO_COMMENT_USE_IP); + // $rdobj->del($this->task_key); + // $rdobj->del(_RS_SPIDER_WEIBO_COMMENT); exit; } diff --git a/tools/filter_comments.py b/tools/filter_comments.py index cadbd99..16483ac 100644 --- a/tools/filter_comments.py +++ b/tools/filter_comments.py @@ -4,6 +4,7 @@ from snownlp import SnowNLP import pymysql import configparser import os +import re def get_db_config(): config = configparser.ConfigParser() @@ -381,6 +382,13 @@ def filter_medical_comments(comments_list): update_records.append((0, comment_id)) continue + img_pattern = re.compile(r'') + commentRemove = re.sub(img_pattern, '', comment) + commentRemove = re.sub(r'\s', '', commentRemove) + if len(commentRemove) <= 4: + update_records.append((0, comment_id)) + continue + s = SnowNLP(comment) sentiment_score = s.sentiments diff --git a/tools/set_weibo_login_cookie.php b/tools/set_weibo_login_cookie.php new file mode 100644 index 0000000..4fdd811 --- /dev/null +++ b/tools/set_weibo_login_cookie.php @@ -0,0 +1,17 @@ +initRedis(); + +$cookies[] = '_2A25FcbH_DeRhGeFG71sY9CbOyDuIHXVmD0s3rDV8PUNbmtB-LROgkW9NeUknrDWZVELgZ5_bHazkmfeqimpsJvrC'; // 吴龙超 +$cookies[] = '_2A25Fcc-2DeRhGeRJ6VAW9SvIzjiIHXVmD01-rDV8PUNbmtAYLRjzkW9NUqgeP0nu8ldMw0ltx31xfvHmVmqDM2xN'; // 王朋达 +$cookies[] = '_2A25FakNADeRhGedJ71UQ9S7PzjyIHXVmBtqIrDV8PUJbkNB-LRX_kW1NVhe7zEjMdJLzXvbT9ck1Q-L9YYtLiCpV'; // 张琪 +$cookies[] = '_2AkMfKU18f8NxqwFRmvoVzmPrZI11wwvEieKpdbynJRMxHRl-yT9xqmoitRB6NKljk7EQIbC2ibY-BOeJTItlu8LSPOXb'; // 刘梦琪 +$cookies[] = '_2A25FcbInDeRhGeFK4lEU9ijJwzSIHXVmD0vvrDV8PUNbmtAYLVLTkW9NQtaHPZxefLemxY2kP3YN6l7tyfSJeWvl'; // 张宇鹏 + +$rdobj->set(_RC_WEIBO_LOGIN_COOKIE2, json_encode($cookies)); +exit; \ No newline at end of file diff --git a/tools/test.py b/tools/test.py new file mode 100644 index 0000000..7a5ce9c --- /dev/null +++ b/tools/test.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +from snownlp import seg +from snownlp import SnowNLP +import pymysql +import configparser +import os +import re + +comment = ' [赞] [赞] ' + +non_img_pattern = re.compile(r'') +comment = re.sub(non_img_pattern, '', comment) +comment = re.sub(r'\s', '', comment) + +print(len(comment)) +print(comment) + diff --git a/view/css/index.css b/view/css/index.css index d77a14a..7b51bbf 100644 --- a/view/css/index.css +++ b/view/css/index.css @@ -494,7 +494,9 @@ button:hover { } .green, -.blue { +.blue, +.red, +.grey { padding: 2px 10px; border-radius: 4px; opacity: 1; @@ -510,6 +512,16 @@ button:hover { background: #F2F7FF; border: 1px solid #BFDAFF; } +.red { + color: #e91621; + background: #F2F7FF; + border: 1px solid #e91621; +} +.grey{ + color: #626573; + background: #F2F7FF; + border: 1px solid #626573; +} .form-item{ display: flex; align-items: normal; diff --git a/view/js/index.js b/view/js/index.js index 1ec0506..d5b0df3 100644 --- a/view/js/index.js +++ b/view/js/index.js @@ -3,6 +3,7 @@ $(document).ready(function () { var tabActive = '1'; var cur_editdata = {}; //当前编辑数据 var vediourl = '' + var hidecomments = $('#hidecomments').length $('.radio-wrap').on('click', '.radio_box input', function () { var radio_val = $(this).attr('value') @@ -450,15 +451,24 @@ $(document).ready(function () { var totalPages = 0; function getRenderData(){ + var param = { + status, + currentPage, + pageSize + } + if(hidecomments == 1) { + var param = { + currentPage, + pageSize, + spider_comment_status:2 + } + } + $.ajax({ type: "post", dataType: 'json', url: '/weibo/ajax_weibo_list', - data: { - status, - currentPage, - pageSize - }, + data: param, success: function (res) { if(!res.status){ alert(`请求失败`); @@ -509,11 +519,12 @@ $(document).ready(function () { ` + item.name + ` 录入：` + item.created_at + ` ${item.wid ? `微博地址` :''} + ${item.wid && hidecomments == 1 ? `查看详情` :''} ${ - tabActive == '1' ? `
编辑
` :'' + tabActive == '1' && hidecomments == 0 ? `
编辑
` :'' } - +

${item.text} @@ -523,9 +534,9 @@ $(document).ready(function () { ${imagesHTML}

- ${tabActive == '1' || tabActive == '2' ? '' : ''} - ${tabActive == '1' ? '' : ''} - ${tabActive == '3' || tabActive == '2' ? '' : ''} + ${((tabActive == '1' || tabActive == '2') && hidecomments == 0) ? '' : ''} + ${(tabActive == '1' && hidecomments == 0) ? '' : ''} + ${((tabActive == '3' || tabActive == '2') && hidecomments == 0) ? '' : ''}
` $('#data-list').html(html1); diff --git a/view/templates/include/header.html b/view/templates/include/header.html index deaaa29..54cc9d0 100644 --- a/view/templates/include/header.html +++ b/view/templates/include/header.html @@ -12,6 +12,9 @@ {if $_user_info.is_super_admin} 子账号管理 {/if} + {if $_user_info.is_super_admin} + 微博评论 + {/if} 退出 diff --git a/view/templates/weibo/comment_detail.html b/view/templates/weibo/comment_detail.html new file mode 100644 index 0000000..6ece0f1 --- /dev/null +++ b/view/templates/weibo/comment_detail.html @@ -0,0 +1,210 @@ + + + + + + + 知识库 + + + + {literal} + + {/literal} + + + +
+ {include file="include/header.html"} +
+
+
    +
  • 选中评论
    • +
  • 剔除评论
    • +
  • 全部评论
    • +
+
+ +
+
微博内容：
+ +
+
+ 微博 + + 张宝旬 + 录入：{$data.created_at} + 微博地址 +
+
+ +
+ {$data.text} +
+ + {if $data.pic_arr} +
+ {foreach from=$data.pic_arr item=item} +
+ 图片 + Small Image + Small Image +
+ {/foreach} +
+ {/if} + + {if $data.video_url} +
+ 视频 + Video Cover + ...丢了 +
+ {/if} + + {if $comment_list} +
+
评论内容：
+ {foreach from=$comment_list key=key item=item} +
+
+ + + {$key+1}.{$item.screen_name} + {$item.comment_time} {$item.source} +
+
+ +
+
+ 评论 + + {$item.content} + {$search_status_list[$item.is_search]} +
+
+ {/foreach} +
+ {/if} +
+ + + +
+ {include file="include/footer.html"} +
+ + + + + + +{literal} + +{/literal} + + \ No newline at end of file diff --git a/view/templates/weibo/comments.html b/view/templates/weibo/comments.html new file mode 100644 index 0000000..2d5700c --- /dev/null +++ b/view/templates/weibo/comments.html @@ -0,0 +1,170 @@ + + + + + + + 知识库 + + + + + + +
+ {include file="include/header.html"} + +
+
+
+ + +
+ + +
+
+ + + + + + + + + +
+

预览

+ Large Image + × +
+ + +
+ + +
+ + +
+
+
+ 编辑 + +
+ +
+
+ +
+ 录入形式 +
+
+
+ +
+ +
+
+ +
+ +
+
+
+ + +
+
+ 回答信息 +
+
+
+ +
0/200
+
+
+
+
+ {literal} + + {/literal} + +
+ +
+
+ + +
+

支持 jpg、png、mp4 格式，单个文件不超过 10MB

+
+ + +
+ + +
+
+
+
+ {include file="include/footer.html"} +
+ + + + + + + \ No newline at end of file