# -*- coding: utf-8 -*- """ @Time : 2023/3/29 14:27 @Author : @FileName: @Software: @Describe: """ import os from flask import Flask, jsonify, Response from flask import request import redis import uuid import json import time import threading from threading import Thread from flask import send_file, send_from_directory import os from flask import make_response import openai import base64 import re import urllib.parse as pa pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50, db=1, password='Zhicheng123*') redis_ = redis.Redis(connection_pool=pool, decode_responses=True) db_key_query = 'query' db_key_querying = 'querying' batch_size = 32 app = Flask(__name__) app.config["JSON_AS_ASCII"] = False import logging lock = threading.RLock() mulu_prompt = "请帮我根据题目为“{}”生成一个论文目录" first_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的大标题“{}”的内容续写完整，保证续写内容不少于800字" small_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的小标题“{}”的内容续写完整，保证续写内容不少于800字" references_prompt = "论文题目是“{}”，目录是“{}”，请为这篇论文生成15篇中文的{}，要求其中有有中文参考文献不低于12篇，英文参考文献不低于2篇" thank_prompt = "论文题目是“{}”，目录是“{}”，请把其中的{}部分续写完整" kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义，要求不少于1500字" chinese_abstract_prompt = "请以《{}》为题目生成论文摘要，要求不少于500字" english_abstract_prompt = "请把“{}”这段文字翻译成英文" chinese_keyword_prompt = "请为“{}”这段论文摘要生成3-5个关键字" english_keyword_prompt = "请把“{}”这几个关键字翻译成英文" thanks = "致谢" dabiaoti = ["二","三","四","五","六","七","八","九"] # 正则 pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+' pantten_other_biaoti = '[2-9二三四五六七八九ⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5]+' project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" api_key_list = ["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", "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", "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", "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", "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", "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", "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", "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", "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"] # "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" def chat_title(title, api_key): global lock # time.sleep(5) # return [str(i) for i in range(20)] openai.api_key = api_key res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": mulu_prompt.format(title)}, ], temperature=0.5 ) lock.acquire() api_key_list.append(api_key) lock.release() mulu = res.choices[0].message.content mulu_list = str(mulu).split("\n") mulu_list = [i.strip() for i in mulu_list if i != ""] return mulu, mulu_list def chat_kaitibaogao(title, api_key, uuid_path): global lock # time.sleep(1) openai.api_key = api_key res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": kaitibaogao_prompt.format(title)}, ], temperature=0.5 ) kaitibaogao = res.choices[0].message.content kaitibaogao_path = os.path.join(uuid_path, "kaitibaogao.txt") with open(kaitibaogao_path, 'w', encoding='utf8') as f_kaitibaogao: f_kaitibaogao.write(kaitibaogao) lock.acquire() api_key_list.append(api_key) lock.release() class GeneratePaper: def __init__(self, mulu, table): self.mulu = mulu self.paper = [""] * len(table) def chat_content_(self,api_key, mulu_title_id, title, mulu, subtitle, prompt): global lock # time.sleep(5) # api_key_list.append(api_key) # self.paper[mulu_title_id] = subtitle if subtitle[:2] == "@@": self.paper[mulu_title_id] = subtitle[2:] else: openai.api_key = api_key res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": prompt.format(title, mulu, subtitle)}, ], temperature=0.5 ) self.paper[mulu_title_id] = res.choices[0].message.content lock.acquire() api_key_list.append(api_key) lock.release() # return res.choices[0].message.content def classify(): # 调用模型，设置最大batch_size while True: if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 time.sleep(3) continue thread_list = [] query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text data_dict_path = json.loads(query) query_id = data_dict_path['id'] title = data_dict_path['title'] # project_data_txt_path = os.path.abspath("new_data_txt") # uuid_path = "new_data_txt/{}/".format(query_id) # uuid路径 uuid_path = os.path.join(project_data_txt_path, query_id) print("uuid",query_id) os.makedirs(uuid_path) print("uuid_path", os.path.exists(uuid_path)) # 生成开题报告 # title, api_key, uuid_path lock.acquire() api_key = api_key_list.pop() lock.release() t = Thread(target=chat_kaitibaogao, args=(title, api_key, uuid_path, )) t.start() thread_list.append(t) # 生成目录 while True: if api_key_list != []: api_key = api_key_list.pop() break else: time.sleep(3) mulu, mulu_list = chat_title(title, api_key) # mulu_base64 = base64.b64encode(mulu.encode('utf-8')) # mulu_path = os.path.join(uuid_path, "mulu.txt") # with open(mulu_path, 'wb', encoding='utf8') as f2: # f2.write(mulu_base64) index = 0 print(mulu_list) cun_bool = False table_of_contents = [mulu_list[0]] for i in mulu_list[1:]: result_second_biaoti_list = re.findall(pantten_second_biaoti, i) result_other_biaoti_list = re.findall(pantten_other_biaoti, i) if result_second_biaoti_list != []: table_of_contents.append("@@" + i) cun_bool = True continue if cun_bool == False: continue else: if result_other_biaoti_list != []: table_of_contents.append("@@" + i) else: table_of_contents.append(i) print(table_of_contents) # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] # print(table_of_contents) thanks_bool_table = table_of_contents[-3:] if thanks not in thanks_bool_table: table_of_contents.insert(-1, "致谢") chat_class = GeneratePaper(mulu_list, table_of_contents) print(len(table_of_contents)) ############################################################ while True: if api_key_list == []: time.sleep(1) continue if index == len(table_of_contents): break lock.acquire() api_key = api_key_list.pop() lock.release() subtitle = table_of_contents[index] if index == 0: prompt = first_title_prompt elif subtitle == "参考文献": prompt = references_prompt elif subtitle == "致谢": prompt = thank_prompt else: prompt = small_title_prompt print("请求的所有参数", api_key, index, title, subtitle, prompt) t = Thread(target=chat_class.chat_content_, args=(api_key, index, title, mulu, subtitle, prompt)) t.start() thread_list.append(t) lock.acquire() index += 1 lock.release() for thread in thread_list: thread.join() print(chat_class.paper) paper = "\n".join(chat_class.paper) print(paper) content_path = os.path.join(uuid_path, "content.txt") with open(content_path, 'w', encoding='utf8') as f_content: f_content.write(paper) mulu_path = os.path.join(uuid_path, "mulu.txt") with open(mulu_path, 'w', encoding='utf8') as f_mulu: f_mulu.write(mulu) kaitibaogao_txt_path = os.path.join(uuid_path, "kaitibaogao.txt") # word保存路径 save_word_paper = os.path.join(uuid_path, "paper.docx") save_word_paper_start = os.path.join(uuid_path, "paper_start.docx".format(title)) # content_base64 = base64.b64encode(paper.encode('utf-8')) # content_path = os.path.join(uuid_path, "content.txt") # with open(content_path, 'wb', encoding='utf8') as f2: # f2.write(content_base64) # 拼接成word title = pa.quote(title) mulu_path = mulu_path content_path = content_path # 调用jar包 print("java_path", mulu_path, content_path, title, save_word_paper) os.system( "java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/createAiXieZuoWord.jar' '{}' '{}' '{}' '{}'".format( mulu_path, content_path, title, save_word_paper)) print("jaba_kaitibaogao", kaitibaogao_txt_path, save_word_paper_start) os.system("java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/createAiXieZuoKaitiWord.jar' '{}' '{}'".format( kaitibaogao_txt_path, save_word_paper_start)) url_path_paper = "http://104.244.89.190:14000/download?filename_path={}/paper.docx".format(query_id) url_path_kaiti = "http://104.244.89.190:14000/download?filename_path={}/paper_start.docx".format(query_id) # content_path = os.path.join(uuid_path, "content.txt") # load_result_path = res_path.format(query_id) # load_result_path = os.path.abspath(load_result_path) # with open(load_result_path, 'w', encoding='utf8') as f2: # f2.write(paper) return_text = str({"id":query_id, "content_url_path": url_path_paper, "content_report_url_path": url_path_kaiti, "probabilities": None, "status_code": 200}) redis_.srem(db_key_querying, query_id) redis_.set(query_id, return_text, 28800) @app.route("/chat", methods=["POST"]) def chat(): print(request.remote_addr) title = request.json["title"] id_ = str(uuid.uuid1()) redis_.rpush(db_key_query, json.dumps({"id":id_, "title": title})) # 加入redis return_text = {"texts": {'id': id_,}, "probabilities": None, "status_code": 200} print("ok") redis_.sadd(db_key_querying, id_) return jsonify(return_text) # 返回结果 @app.route("/download", methods=['GET']) def download_file(): # 需要知道2个参数, 第1个参数是本地目录的path, 第2个参数是文件名(带扩展名) # directory = os.path.join(project_data_txt_path, filename) # 假设在当前目录 # uuid_path, word_name = str(filename).split("/") # word_path_root = os.path.join(project_data_txt_path, uuid_path) # response = make_response(send_from_directory(word_path_root, word_name, as_attachment=True)) # response.headers["Content-Disposition"] = "attachment; filename={}".format(filename.encode().decode('latin-1')) filename_path = request.args.get('filename_path', '') filename = filename_path.split("/")[1] path_name = os.path.join(project_data_txt_path, filename_path) with open(path_name, 'rb') as f: stream = f.read() response = Response(stream, content_type='application/octet-stream') response.headers['Content-disposition'] = 'attachment; filename={}'.format(filename) return response @app.route("/search", methods=["POST"]) def search(): id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" result = redis_.get(id_) # 获取该query的模型结果 if result is not None: # redis_.delete(id_) # result_dict = result.decode('UTF-8') result_dict = eval(result) # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} query_id = result_dict["id"] # "content_url_path": url_path_paper, # "content_report_url_path": url_path_kaiti, content_url_path = result_dict["content_url_path"] content_report_url_path = result_dict["content_report_url_path"] probabilities = result_dict["probabilities"] result_text = {'code': 200, 'content_url_path': content_url_path, 'content_report_url_path': content_report_url_path, 'probabilities': probabilities} else: querying_list = list(redis_.smembers("querying")) querying_set = set() for i in querying_list: querying_set.add(i.decode()) querying_bool = False if id_ in querying_set: querying_bool = True query_list_json = redis_.lrange(db_key_query, 0, -1) query_set_ids = set() for i in query_list_json: data_dict = json.loads(i) query_id = data_dict['id'] query_set_ids.add(query_id) query_bool = False if id_ in query_set_ids: query_bool = True if querying_bool == True and query_bool == True: result_text = {'code': "201", 'text': "", 'probabilities': None} elif querying_bool == True and query_bool == False: result_text = {'code': "202", 'text': "", 'probabilities': None} else: result_text = {'code': "203", 'text': "", 'probabilities': None} return jsonify(result_text) # 返回结果 t1 = Thread(target=classify) t1.start() if __name__ == "__main__": app.run(host="0.0.0.0", port=14000, threaded=True, debug=False)