jiahui
/
generate_articles_directory
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

增加期刊目录

master
majiahui@haimaqingfan.com 3 months ago
parent
27b48310ea
commit
130fca354b
2 changed files with 464 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 232
      articles_directory_predict_qikan_word.py
  2. 232
      articles_directory_predict_qikan_word_online.py

232
articles_directory_predict_qikan_word.py
View File

@ -0,0 +1,232 @@
from flask import Flask, jsonify
from flask import request
import time
import concurrent.futures
import requests
import socket
import os
class log:
def __init__(self):
pass
def log(*args, **kwargs):
format = '%Y/%m/%d-%H:%M:%S'
format_h = '%Y-%m-%d'
value = time.localtime(int(time.time()))
dt = time.strftime(format, value)
dt_log_file = time.strftime(format_h, value)
log_file = 'log_file/access-%s' % dt_log_file + ".log"
if not os.path.exists(log_file):
with open(os.path.join(log_file), 'w', encoding='utf-8') as f:
print(dt, *args, file=f, **kwargs)
else:
with open(os.path.join(log_file), 'a+', encoding='utf-8') as f:
print(dt, *args, file=f, **kwargs)
def get_host_ip():
"""
查询本机ip地址
:return: ip
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
finally:
s.close()
return ip
chatgpt_url_predict = "http://{}:12000/predict".format(str(get_host_ip()))
chatgpt_url_search = "http://{}:12000/search".format(str(get_host_ip()))
prompt = {
"mulu_title_Level_2_1": "<|im_start|>user\n任务：生成目录\n请帮我生成一个期刊论文的目录，论文题目是“%s”，要求只有一级标题，二级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；一级标题生成%s个；每个一级标题包含%s-%s个二级标题<|im_end|>\n<|im_start|>assistant\n",
"mulu_title_Level_3_1": "<|im_start|>user\n任务：生成目录\n请帮我生成一个期刊论文的目录，论文题目是“%s”，要求只有一级标题，二级标题和三级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；三级标题使用阿拉伯数字 例如1.1.2 xxx；一级标题生成%s个；每个一级标题包含%s-%s个二级标题；三级标题个数适中<|im_end|>\n<|im_start|>assistant\n",
}
nums_word_dict = {
"3000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "3",
"title_2_nums": ["0","3"],
"title_small_nums": ["3", "0","3"]
},
"5000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "4",
"title_2_nums": ["0","5"],
"title_small_nums": ["4", "0","5"]
},
"8000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "5",
"title_2_nums": ["0", "6"],
"title_small_nums": ["5", "0", "6"]
},
"10000": {
"prompt": prompt["mulu_title_Level_3_1"],
"title_1_nums": "6",
"title_2_nums": ["0", "5"],
"title_small_nums": ["6", "0", "5"],
}
}
app = Flask(__name__)
app.config["JSON_AS_ASCII"] = False
def dialog_line_parse(url, text):
"""
将数据输入模型进行分析并输出结果
:param url: 模型url
:param text: 进入模型的数据
:return: 模型返回结果
"""
response = requests.post(
url,
json=text,
timeout=1000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("【{}】 Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(url, response.status_code, response.text))
print(text)
return []
def request_api_chatgpt(prompt):
'''
content = request.json["content"] # 获取用户query中的文本 例如"I love you"
model = request.json["model"]
top_p = request.json["top_p"]
temperature = request.json["temperature"]
:param prompt:
:return:
'''
data = {
"content": prompt,
"model": "gpt-4-turbo",
"top_p": 0.95,
"temperature": 0.7
}
response = requests.post(
chatgpt_url_predict,
json=data,
timeout=100000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(response.status_code, response.text))
return {}
def uuid_search(uuid):
data = {
"id": uuid
}
response = requests.post(
chatgpt_url_search,
json=data,
timeout=100000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(response.status_code, response.text))
return {}
def uuid_search_mp(results):
results_list = [""] * len(results)
while True:
tiaochu_bool = True
for i in results_list:
if i == "":
tiaochu_bool = False
break
if tiaochu_bool == True:
break
for i in range(len(results)):
uuid = results[i]["texts"]["id"]
result = uuid_search(uuid)
if result["code"] == 200:
results_list[i] = result["text"]
time.sleep(3)
return results_list
def get_multiple_urls(urls):
input_values = []
for i in urls:
input_values.append(i[1])
with concurrent.futures.ThreadPoolExecutor() as executor:
# 使用map方法并发地调用worker_function
results = list(executor.map(request_api_chatgpt, input_values))
with concurrent.futures.ThreadPoolExecutor() as executor:
# 使用map方法并发地调用worker_function
results = list(executor.map(uuid_search_mp, [results]))
return_list = []
for i in results[0]:
data = {
"code": 200,
"data": i
}
return_list.append(data)
return return_list
@app.route("/articles_directory", methods=["POST"])
def articles_directory():
title = request.json["title"] # 获取用户query中的文本 例如"I love you"
nums_catalogue = request.json["nums_catalogue"]
nums_word = request.json["nums_word"] # 获取用户query中的文本 例如"I love you"
nums_catalogue = int(nums_catalogue)
keyword = tuple([title] + nums_word_dict[nums_word]["title_small_nums"])
prompt = nums_word_dict[nums_word]["prompt"]%keyword
print(prompt)
input_data = []
for i in range(nums_catalogue):
input_data.append([i, prompt])
results = get_multiple_urls(input_data)
log.log('text:{},'.format(prompt))
return jsonify(results) # 返回结果
if __name__ == "__main__":
app.run(debug=False, host='0.0.0.0', port=18004)

232
articles_directory_predict_qikan_word_online.py
View File

@ -0,0 +1,232 @@
from flask import Flask, jsonify
from flask import request
import time
import concurrent.futures
import requests
import socket
import os
class log:
def __init__(self):
pass
def log(*args, **kwargs):
format = '%Y/%m/%d-%H:%M:%S'
format_h = '%Y-%m-%d'
value = time.localtime(int(time.time()))
dt = time.strftime(format, value)
dt_log_file = time.strftime(format_h, value)
log_file = 'log_file/access-%s' % dt_log_file + ".log"
if not os.path.exists(log_file):
with open(os.path.join(log_file), 'w', encoding='utf-8') as f:
print(dt, *args, file=f, **kwargs)
else:
with open(os.path.join(log_file), 'a+', encoding='utf-8') as f:
print(dt, *args, file=f, **kwargs)
def get_host_ip():
"""
查询本机ip地址
:return: ip
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
ip = s.getsockname()[0]
finally:
s.close()
return ip
chatgpt_url_predict = "http://{}:12001/predict".format(str(get_host_ip()))
chatgpt_url_search = "http://{}:12001/search".format(str(get_host_ip()))
prompt = {
"mulu_title_Level_2_1": "<|im_start|>user\n任务：生成目录\n请帮我生成一个期刊论文的目录，论文题目是“%s”，要求只有一级标题，二级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；一级标题生成%s个；每个一级标题包含%s-%s个二级标题<|im_end|>\n<|im_start|>assistant\n",
"mulu_title_Level_3_1": "<|im_start|>user\n任务：生成目录\n请帮我生成一个期刊论文的目录，论文题目是“%s”，要求只有一级标题，二级标题和三级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；三级标题使用阿拉伯数字 例如1.1.2 xxx；一级标题生成%s个；每个一级标题包含%s-%s个二级标题；三级标题个数适中<|im_end|>\n<|im_start|>assistant\n",
}
nums_word_dict = {
"3000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "3",
"title_2_nums": ["0","3"],
"title_small_nums": ["3", "0","3"]
},
"5000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "4",
"title_2_nums": ["0","5"],
"title_small_nums": ["4", "0","5"]
},
"8000": {
"prompt": prompt["mulu_title_Level_2_1"],
"title_1_nums": "5",
"title_2_nums": ["0", "6"],
"title_small_nums": ["5", "0", "6"]
},
"10000": {
"prompt": prompt["mulu_title_Level_3_1"],
"title_1_nums": "6",
"title_2_nums": ["0", "5"],
"title_small_nums": ["6", "0", "5"],
}
}
app = Flask(__name__)
app.config["JSON_AS_ASCII"] = False
def dialog_line_parse(url, text):
"""
将数据输入模型进行分析并输出结果
:param url: 模型url
:param text: 进入模型的数据
:return: 模型返回结果
"""
response = requests.post(
url,
json=text,
timeout=1000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("【{}】 Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(url, response.status_code, response.text))
print(text)
return []
def request_api_chatgpt(prompt):
'''
content = request.json["content"] # 获取用户query中的文本 例如"I love you"
model = request.json["model"]
top_p = request.json["top_p"]
temperature = request.json["temperature"]
:param prompt:
:return:
'''
data = {
"content": prompt,
"model": "gpt-4-turbo",
"top_p": 0.95,
"temperature": 0.7
}
response = requests.post(
chatgpt_url_predict,
json=data,
timeout=100000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(response.status_code, response.text))
return {}
def uuid_search(uuid):
data = {
"id": uuid
}
response = requests.post(
chatgpt_url_search,
json=data,
timeout=100000
)
if response.status_code == 200:
return response.json()
else:
# logger.error(
# "【{}】 Failed to get a proper response from remote "
# "server. Status Code: {}. Response: {}"
# "".format(url, response.status_code, response.text)
# )
print("Failed to get a proper response from remote "
"server. Status Code: {}. Response: {}"
"".format(response.status_code, response.text))
return {}
def uuid_search_mp(results):
results_list = [""] * len(results)
while True:
tiaochu_bool = True
for i in results_list:
if i == "":
tiaochu_bool = False
break
if tiaochu_bool == True:
break
for i in range(len(results)):
uuid = results[i]["texts"]["id"]
result = uuid_search(uuid)
if result["code"] == 200:
results_list[i] = result["text"]
time.sleep(3)
return results_list
def get_multiple_urls(urls):
input_values = []
for i in urls:
input_values.append(i[1])
with concurrent.futures.ThreadPoolExecutor() as executor:
# 使用map方法并发地调用worker_function
results = list(executor.map(request_api_chatgpt, input_values))
with concurrent.futures.ThreadPoolExecutor() as executor:
# 使用map方法并发地调用worker_function
results = list(executor.map(uuid_search_mp, [results]))
return_list = []
for i in results[0]:
data = {
"code": 200,
"data": i
}
return_list.append(data)
return return_list
@app.route("/articles_directory", methods=["POST"])
def articles_directory():
title = request.json["title"] # 获取用户query中的文本 例如"I love you"
nums_catalogue = request.json["nums_catalogue"]
nums_word = request.json["nums_word"] # 获取用户query中的文本 例如"I love you"
nums_catalogue = int(nums_catalogue)
keyword = tuple([title] + nums_word_dict[nums_word]["title_small_nums"])
prompt = nums_word_dict[nums_word]["prompt"]%keyword
print(prompt)
input_data = []
for i in range(nums_catalogue):
input_data.append([i, prompt])
results = get_multiple_urls(input_data)
log.log('text:{},'.format(prompt))
return jsonify(results) # 返回结果
if __name__ == "__main__":
app.run(debug=False, host='0.0.0.0', port=18005)
Write Preview
Loading…
Cancel
Save