majiahui@haimaqingfan.com
8 months ago
8 changed files with 546 additions and 18 deletions
@ -0,0 +1,14 @@ |
|||||
|
## 参考文献生成 |
||||
|
新参考文件项目使用faiss聚类方式查询，实现秒级响应 |
||||
|
|
||||
|
|
||||
|
#### 生成ndarray数据，生成文献基本信息 |
||||
|
修改或增加数据 |
||||
|
python 数据生成ndarray.py |
||||
|
|
||||
|
#### 训练faiss |
||||
|
修改或增加 npy文件路径 |
||||
|
python 训练faiss.py |
||||
|
|
||||
|
#### 部署服务 |
||||
|
python generate_references_api.py |
||||
@ -0,0 +1,236 @@ |
|||||
|
import os |
||||
|
import random |
||||
|
|
||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = "0" |
||||
|
from flask import Flask, jsonify |
||||
|
from flask import request |
||||
|
import numpy as np |
||||
|
import faiss |
||||
|
import json |
||||
|
import requests |
||||
|
import socket |
||||
|
from sentence_transformers import SentenceTransformer |
||||
|
|
||||
|
|
||||
|
with open("data/lable/id2lable.json", encoding="utf-8") as f: |
||||
|
id2lable = json.loads(f.read()) |
||||
|
|
||||
|
with open("data/lable/lable2id.json", encoding="utf-8") as f: |
||||
|
lable2id = json.loads(f.read()) |
||||
|
|
||||
|
with open("data/discipline_types.json") as f: |
||||
|
lable_discipline_types = json.loads(f.read()) |
||||
|
|
||||
|
|
||||
|
app = Flask(__name__) |
||||
|
app.config["JSON_AS_ASCII"] = False |
||||
|
|
||||
|
d = 768 # dimension |
||||
|
model = SentenceTransformer('Dmeta-embedding-zh') |
||||
|
|
||||
|
def get_host_ip(): |
||||
|
""" |
||||
|
查询本机ip地址 |
||||
|
:return: ip |
||||
|
""" |
||||
|
try: |
||||
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) |
||||
|
s.connect(('8.8.8.8', 80)) |
||||
|
ip = s.getsockname()[0] |
||||
|
finally: |
||||
|
s.close() |
||||
|
|
||||
|
return ip |
||||
|
|
||||
|
# url = "http://{}:50003/roformer".format(str(get_host_ip())) |
||||
|
url = "http://{}:50003/roformer".format("192.168.31.149") |
||||
|
|
||||
|
def dialog_line_parse(url, text): |
||||
|
""" |
||||
|
将数据输入模型进行分析并输出结果 |
||||
|
:param url: 模型url |
||||
|
:param text: 进入模型的数据 |
||||
|
:return: 模型返回结果 |
||||
|
""" |
||||
|
|
||||
|
response = requests.post( |
||||
|
url, |
||||
|
json=text, |
||||
|
timeout=1000 |
||||
|
) |
||||
|
if response.status_code == 200: |
||||
|
return response.json() |
||||
|
else: |
||||
|
# logger.error( |
||||
|
# "【{}】 Failed to get a proper response from remote " |
||||
|
# "server. Status Code: {}. Response: {}" |
||||
|
# "".format(url, response.status_code, response.text) |
||||
|
# ) |
||||
|
print("【{}】 Failed to get a proper response from remote " |
||||
|
"server. Status Code: {}. Response: {}" |
||||
|
"".format(url, response.status_code, response.text)) |
||||
|
print(text) |
||||
|
return [] |
||||
|
|
||||
|
|
||||
|
def panduan_paper_lable(paper_lable_text): |
||||
|
paper_lable = { |
||||
|
"硕士": "D", |
||||
|
"期刊": "J", |
||||
|
"博士": "J" |
||||
|
} |
||||
|
return paper_lable[paper_lable_text] |
||||
|
|
||||
|
|
||||
|
def ulit_recall_paper(reference_list, nums): |
||||
|
''' |
||||
|
对返回的十篇文章路径读取并解析 |
||||
|
:param recall_data_list_path: |
||||
|
:return data: list [[sentence, filename],[sentence, filename],[sentence, filename]] |
||||
|
''' |
||||
|
|
||||
|
# data = [] |
||||
|
# for path in recall_data_list_path: |
||||
|
# filename = path.split("/")[-1] |
||||
|
# with open(path, encoding="gbk") as f: |
||||
|
# text = f.read() |
||||
|
# text_list = text.split("\n") |
||||
|
# for sentence in text_list: |
||||
|
# if sentence != "": |
||||
|
# data.append([sentence, filename]) |
||||
|
# return data |
||||
|
|
||||
|
# recall_data_list |
||||
|
# 作者 论文名称 论文类别 论文来源 论文年份 摘要 期刊 |
||||
|
# "[1]赵璐.基于旅游资源开发下的新农村景观营建研究[D].西安建筑科技大学,2014." |
||||
|
|
||||
|
data_info = [] |
||||
|
data_title = [] |
||||
|
|
||||
|
for data_one in reference_list: |
||||
|
|
||||
|
if data_one[1] not in data_title: |
||||
|
|
||||
|
print("data_one", data_one) |
||||
|
print("data_one[0]", data_one[0]) |
||||
|
paper = ".".join([ |
||||
|
",".join([str(i).replace("\n", "").replace("\r", "") for i in data_one[0].split(";") if i != ""]), |
||||
|
data_one[1] + f"[{panduan_paper_lable(data_one[6])}]", |
||||
|
",".join([ |
||||
|
data_one[3], str(data_one[4]) + "." |
||||
|
]) |
||||
|
]) |
||||
|
|
||||
|
data_title.append(data_one[1]) |
||||
|
data_info.append({ |
||||
|
"author": data_one[0], |
||||
|
"title": data_one[1], |
||||
|
"special_topic": data_one[2], |
||||
|
"qikan_name": data_one[3], |
||||
|
"year": str(data_one[4]), |
||||
|
"abstract": data_one[5], |
||||
|
"classlable": data_one[6], |
||||
|
"reference": paper |
||||
|
}) |
||||
|
|
||||
|
# print(data) |
||||
|
print(data_title) |
||||
|
print(nums) |
||||
|
random.shuffle(data_info) |
||||
|
random.shuffle(data_info) |
||||
|
data_info = data_info[:int(nums)] |
||||
|
return data_info |
||||
|
|
||||
|
|
||||
|
def main(title, abstract, nums): |
||||
|
data = { |
||||
|
"title": title, |
||||
|
"abst_zh": abstract, |
||||
|
"content": "" |
||||
|
} |
||||
|
# { |
||||
|
# "label_num": [ |
||||
|
# 117, |
||||
|
# 143 |
||||
|
# ] |
||||
|
# } |
||||
|
result = dialog_line_parse(url, data) |
||||
|
|
||||
|
# print(result['label_num'][0]) |
||||
|
# print(id2lable[result['label_num'][0]]) |
||||
|
subject_pinyin = lable_discipline_types[id2lable[str(result['label_num'][0])]] |
||||
|
|
||||
|
# with open(f"data/prompt/{subject_pinyin}.npy") as : |
||||
|
# zidonghua = np.load('data/prompt/{subject_pinyin}.npy') |
||||
|
|
||||
|
data_subject = np.load(f"data/prompt_qikan/{subject_pinyin}.npy") |
||||
|
data_subject_1 = np.load(f"data/prompt_master/{subject_pinyin}.npy") |
||||
|
data_subject_2 = np.load(f"data/prompt_doctor/{subject_pinyin}.npy") |
||||
|
print("xb.shape", data_subject.shape) |
||||
|
print("xb_1.shape", data_subject_1.shape) |
||||
|
print("xb_2.shape", data_subject_2.shape) |
||||
|
data_subject = np.concatenate((data_subject, data_subject_1, data_subject_2)) |
||||
|
print("data_subject.shape", data_subject.shape) |
||||
|
|
||||
|
index = faiss.read_index(f'data/prompt_qikan_master_doctor_ivf/{subject_pinyin}.ivf') |
||||
|
|
||||
|
with open(f"data/data_info_qikan/{subject_pinyin}.json") as f: |
||||
|
data_info = json.loads(f.read()) |
||||
|
|
||||
|
with open(f"data/data_info_master/{subject_pinyin}.json") as f: |
||||
|
data_info_1 = json.loads(f.read()) |
||||
|
|
||||
|
with open(f"data/data_info_doctor/{subject_pinyin}.json") as f: |
||||
|
data_info_2 = json.loads(f.read()) |
||||
|
|
||||
|
print(len(data_info)) |
||||
|
print(len(data_info_1)) |
||||
|
print(len(data_info_2)) |
||||
|
data_info = data_info + data_info_1 + data_info_2 |
||||
|
print(len(data_info)) |
||||
|
print(data_info[0]) |
||||
|
index.add(data_subject) |
||||
|
# index.nprobe = 2 # default nprobe is 1, try a few more |
||||
|
# k = nums |
||||
|
k = 20 |
||||
|
prompt = "标题：“{}”，摘要：“{}”".format(title, abstract) |
||||
|
embs = model.encode([prompt], normalize_embeddings=True) |
||||
|
|
||||
|
D, I = index.search(embs, int(k)) |
||||
|
# print(I) |
||||
|
|
||||
|
reference_list = [] |
||||
|
for i in I[0]: |
||||
|
reference_list.append(data_info[i]) |
||||
|
|
||||
|
data_info = ulit_recall_paper(reference_list, nums) |
||||
|
return "200", data_info |
||||
|
|
||||
|
|
||||
|
@app.route("/", methods=["POST"]) |
||||
|
def handle_query(): |
||||
|
# try: |
||||
|
title = request.form.get("title") |
||||
|
abstract = "" |
||||
|
nums = request.form.get('nums') |
||||
|
|
||||
|
# content = ulit_request_file(file) |
||||
|
|
||||
|
status_code, data_info_list = main(title, abstract, nums) |
||||
|
|
||||
|
if status_code == "400": |
||||
|
return_text = {"resilt": "", "probabilities": None, "status_code": 400} |
||||
|
else: |
||||
|
if status_code == "200": |
||||
|
return_text = { |
||||
|
"data_info": data_info_list, |
||||
|
"probabilities": None, |
||||
|
"status_code": 200 |
||||
|
} |
||||
|
else: |
||||
|
return_text = {"resilt": "", "probabilities": None, "status_code": 400} |
||||
|
|
||||
|
return jsonify(return_text) # 返回结果 |
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
app.run(host="0.0.0.0", port=17003, threaded=True) |
||||
@ -0,0 +1,74 @@ |
|||||
|
import json |
||||
|
from tqdm import tqdm |
||||
|
# json.load() |
||||
|
|
||||
|
# with open("t_xuewei_cnki_spider.csv", encoding="utf-8") as f: |
||||
|
# a = f.read() |
||||
|
# print(a) |
||||
|
|
||||
|
import pandas as pd |
||||
|
|
||||
|
filename = 'data/spider_latest_doctor_paper_list.csv' |
||||
|
chunksize = 10000 # 指定每次读取的行数，可以根据需要调整 |
||||
|
|
||||
|
df_list = [] |
||||
|
# 使用 chunksize 参数迭代读取 CSV 文件 |
||||
|
for chunk in pd.read_csv(filename, chunksize=chunksize): |
||||
|
# 作者 论文名称 论文类别 论文来源 论文年份 摘要 |
||||
|
|
||||
|
# 对每个 chunk 进行处理 |
||||
|
|
||||
|
# print(chunk.columns) |
||||
|
# 9 / 0 |
||||
|
df_list_dan = chunk.values.tolist() |
||||
|
# print(df_list[0]) |
||||
|
for i in tqdm(range(len(df_list_dan))): |
||||
|
if str(df_list_dan[i][2]) != "nan" and \ |
||||
|
str(df_list_dan[i][1]) != "nan" and\ |
||||
|
str(df_list_dan[i][6]) != "nan" and\ |
||||
|
str(df_list_dan[i][3]) != "nan" and\ |
||||
|
str(df_list_dan[i][4]) != "nan" and\ |
||||
|
str(df_list_dan[i][13]) != "nan": |
||||
|
|
||||
|
df_list.append({ |
||||
|
'author': df_list_dan[i][2], |
||||
|
'title': df_list_dan[i][1], |
||||
|
'special_topic': df_list_dan[i][6], |
||||
|
'qikan_name': df_list_dan[i][3], |
||||
|
'year': df_list_dan[i][4], |
||||
|
'abstract': df_list_dan[i][13], |
||||
|
}) |
||||
|
|
||||
|
# data = [] |
||||
|
# json_list = [ |
||||
|
# "/home/majiahui/project/爬取目录筛选/t_xuewei_detail_cnki.json", |
||||
|
# "/home/majiahui/project/爬取目录筛选/t_xuewei_detail_cnki2.json", |
||||
|
# "/home/majiahui/project/爬取目录筛选/t_xuewei_detail_cnki3.json", |
||||
|
# "/home/majiahui/project/爬取目录筛选/t_xuewei_detail_cnki6.json", |
||||
|
# "/home/majiahui/project/爬取目录筛选/t_xuewei_detail_cnki7.json", |
||||
|
# ] |
||||
|
# |
||||
|
# |
||||
|
# print("主库数据完成加载") |
||||
|
# for path in json_list: |
||||
|
# name, typr_file = path.split(".") |
||||
|
# name = name.split("/")[-1] |
||||
|
# a = json.load(open(path)) |
||||
|
# for i in a: |
||||
|
# autoid = "_".join([name, str(i['autoid'])]) |
||||
|
# if autoid in df_dict: |
||||
|
# data.append([i['f_title']] + df_dict[autoid]) |
||||
|
# print("path完成筛选") |
||||
|
# |
||||
|
|
||||
|
print(len(df_list)) |
||||
|
with open("data/data_0423_doctor.json", "w", encoding="utf-8") as f: |
||||
|
f.write(json.dumps(df_list, ensure_ascii=False, indent=2)) |
||||
|
|
||||
|
# |
||||
|
# with open("data.json", encoding="utf-8") as f: |
||||
|
# for i in f.readlines(): |
||||
|
# a = json.loads(i) |
||||
|
# |
||||
|
# |
||||
|
# print(a) |
||||
@ -0,0 +1,134 @@ |
|||||
|
import os |
||||
|
|
||||
|
os.environ["CUDA_VISIBLE_DEVICES"] = "1" |
||||
|
import json |
||||
|
import numpy as np |
||||
|
from tqdm import tqdm |
||||
|
from sentence_transformers import SentenceTransformer |
||||
|
import re |
||||
|
|
||||
|
model = SentenceTransformer('Dmeta-embedding-zh') |
||||
|
print(1) |
||||
|
with open("data/discipline_types.json", encoding="utf-8") as f: |
||||
|
lable_discipline_types = json.loads(f.read()) |
||||
|
|
||||
|
|
||||
|
def erjimul_ulit(): |
||||
|
pass |
||||
|
|
||||
|
|
||||
|
def shengcehng_array(data): |
||||
|
embs = model.encode(data, normalize_embeddings=True) |
||||
|
return embs |
||||
|
|
||||
|
|
||||
|
def is_contain_chinese(word): |
||||
|
""" |
||||
|
判断字符串是否包含中文字符 |
||||
|
:param word: 字符串 |
||||
|
:return: 布尔值，True表示包含中文，False表示不包含中文 |
||||
|
""" |
||||
|
pattern = re.compile(r'[\u4e00-\u9fa5]') |
||||
|
match = pattern.search(word) |
||||
|
return True if match else False |
||||
|
|
||||
|
|
||||
|
if __name__ == '__main__': |
||||
|
|
||||
|
# data = [] |
||||
|
with open("data/data_0423_doctor.json", encoding="utf-8") as f: |
||||
|
# for i in f.readlines(): |
||||
|
# a = json.loads(i) |
||||
|
# data.append(a) |
||||
|
data = json.loads(f.read()) |
||||
|
|
||||
|
print(len(data)) |
||||
|
|
||||
|
a = 0 |
||||
|
|
||||
|
a_ = 0 |
||||
|
data_info = {} # 作者 论文名称 论文类别 论文来源 论文年份 摘要 |
||||
|
data_prompt = {} |
||||
|
|
||||
|
data_info_en = {} # 作者 论文名称 论文类别 论文来源 论文年份 摘要 |
||||
|
data_prompt_en = {} |
||||
|
|
||||
|
for data_dan in data: |
||||
|
if str(data_dan["special_topic"]) == "nan" or \ |
||||
|
str(data_dan["author"]) == "nan" or \ |
||||
|
str(data_dan["title"]) == "nan" or \ |
||||
|
str(data_dan["qikan_name"]) == "nan" or \ |
||||
|
str(data_dan["year"]) == "nan" or \ |
||||
|
str(data_dan["abstract"]) == "nan": |
||||
|
a_ += 1 |
||||
|
continue |
||||
|
|
||||
|
leibie_list = data_dan["special_topic"].split(";") |
||||
|
for leibie in leibie_list: |
||||
|
if leibie in lable_discipline_types: |
||||
|
zh_bool = is_contain_chinese(data_dan["title"]) |
||||
|
|
||||
|
if zh_bool == True: |
||||
|
if lable_discipline_types[leibie] not in data_prompt: |
||||
|
dan_data_prompt = "标题：“{}”，摘要：“{}”".format(data_dan["title"], data_dan["abstract"]) |
||||
|
data_prompt[lable_discipline_types[leibie]] = [dan_data_prompt] |
||||
|
data_info[lable_discipline_types[leibie]] = [ |
||||
|
[data_dan["author"], data_dan["title"], data_dan["special_topic"], data_dan["qikan_name"], |
||||
|
data_dan["year"], data_dan["abstract"], "博士"]] |
||||
|
else: |
||||
|
dan_data_prompt = "标题：“{}”，摘要：“{}”".format(data_dan["title"], data_dan["abstract"]) |
||||
|
data_prompt[lable_discipline_types[leibie]].append(dan_data_prompt) |
||||
|
data_info[lable_discipline_types[leibie]].append( |
||||
|
[data_dan["author"], data_dan["title"], data_dan["special_topic"], data_dan["qikan_name"], |
||||
|
data_dan["year"], data_dan["abstract"], "博士"]) |
||||
|
else: |
||||
|
if lable_discipline_types[leibie] not in data_prompt_en: |
||||
|
dan_data_prompt = "标题：“{}”，摘要：“{}”".format(data_dan["title"], data_dan["abstract"]) |
||||
|
data_prompt_en[lable_discipline_types[leibie]] = [dan_data_prompt] |
||||
|
data_info_en[lable_discipline_types[leibie]] = [ |
||||
|
[data_dan["author"], data_dan["title"], data_dan["special_topic"], data_dan["qikan_name"], |
||||
|
data_dan["year"], data_dan["abstract"], "博士"]] |
||||
|
else: |
||||
|
dan_data_prompt = "标题：“{}”，摘要：“{}”".format(data_dan["title"], data_dan["abstract"]) |
||||
|
data_prompt_en[lable_discipline_types[leibie]].append(dan_data_prompt) |
||||
|
data_info_en[lable_discipline_types[leibie]].append( |
||||
|
[data_dan["author"], data_dan["title"], data_dan["special_topic"], data_dan["qikan_name"], |
||||
|
data_dan["year"], data_dan["abstract"], "博士"]) |
||||
|
|
||||
|
a += 1 |
||||
|
|
||||
|
print(2) |
||||
|
strat = 0 |
||||
|
end = 10000 |
||||
|
print(len(data_prompt)) |
||||
|
for leibie in tqdm(data_prompt): |
||||
|
data_ndarray = np.empty((0, 768)) |
||||
|
print("len(data_prompt[leibie])", len(data_prompt[leibie])) |
||||
|
while True: |
||||
|
if end >= len(data_prompt[leibie]): |
||||
|
break |
||||
|
linshi_data = data_prompt[leibie][strat:end] |
||||
|
data_ndarray = np.concatenate((data_ndarray, shengcehng_array(linshi_data))) |
||||
|
print("data_ndarray.shape", data_ndarray.shape) |
||||
|
strat = end |
||||
|
end += 10000 |
||||
|
|
||||
|
linshi_data = data_prompt[leibie][strat:len(data_prompt[leibie])] |
||||
|
print("len(linshi_data)", len(linshi_data)) |
||||
|
data_ndarray = np.concatenate((data_ndarray, shengcehng_array(linshi_data))) |
||||
|
print("data_ndarray.shape", data_ndarray.shape) |
||||
|
np.save(f'data/prompt_doctor/{leibie}.npy', data_ndarray) |
||||
|
strat = 0 |
||||
|
end = 10000 |
||||
|
|
||||
|
for leibie in data_info: |
||||
|
print(len(data_info[leibie])) |
||||
|
with open(f"data/data_info_doctor/{leibie}.json", "w", encoding="utf-8") as f: |
||||
|
f.write(json.dumps(data_info[leibie], ensure_ascii=False, indent=2)) |
||||
|
|
||||
|
for i in data_prompt_en: |
||||
|
print(i) |
||||
|
print(len(data_prompt_en[i])) |
||||
|
|
||||
|
print(len(data)) |
||||
|
print(a_) |
||||
Loading…
Reference in new issue