import json
import re
from tqdm import tqdm
import os
os . environ [ " CUDA_VISIBLE_DEVICES " ] = " 2,3 "
import torch
from transformers import AutoModelForCausalLM , AutoTokenizer
from transformers . generation . utils import GenerationConfig
model_path = " /home/majiahui/project/models-llm/Baichuan-13B-Chat "
tokenizer = AutoTokenizer . from_pretrained ( model_path , use_fast = False , trust_remote_code = True )
model = AutoModelForCausalLM . from_pretrained ( model_path , device_map = " auto " , torch_dtype = torch . float16 , trust_remote_code = True )
model . generation_config = GenerationConfig . from_pretrained ( model_path )
def predict_baichuan ( text ) :
messages = [ ]
text = " 把下面这段翻译成英文 \n " + text
messages . append ( { " role " : " user " ,
" content " : text } )
response = model . chat ( tokenizer , messages )
return response
prompt_tpye_dict = {
" 生成论文来源的背景# \n " : {
" data_tpye " : " Background for generating paper sources# \n " ,
" title " : " 以《(.*)》为论文题目 " ,
" en-prompt " : " Using ‘ { $title}’ as the title of the paper, write a paragraph about the background of the English question source, with a minimum word count of { $wordnum} words "
} ,
" 生成研究内容# \n " : {
" data_tpye " : " Generate research content# \n " ,
" title " : " 请帮我生成《(.*)》为题目 " ,
" en-prompt " : " Please help me generate the English research content titled ‘ { $title}’, including an overall introduction and a summary of at least three aspects "
} ,
" 生成目录# \n " : {
" data_tpye " : " Generate directory# \n " ,
" title " : " 为论文题目《(.*)》 " ,
" en-prompt " : " Generate an English directory for the paper title ‘ { $title}’, which requires only the first level title and second level title. The format of the first level title is as follows: 1. xxx; The format of the secondary title is as follows: 1.1 xxx; 4 first level titles; Each first level title should contain at least 3 second level titles; "
} ,
" 生成课题的研究背景和意义# \n " : {
" data_tpye " : " Generate directory# \n " ,
" title " : " 请分别写出以《(.*)》为课题 " ,
" en-prompt " : " Please provide the English research background and significance of ‘ { $title}’ as the topic, with a minimum of { $wordnum} words "
} ,
" 生成致谢# \n " : {
" data_tpye " : " Generate acknowledgments# \n " ,
" title " : " 请以《(.*)》为题写一篇论文的中文致谢 " ,
" en-prompt " : " Write an English thank based on the paper title ‘ { $title}’ "
} ,
" 生成论文简短总结# \n " : {
" data_tpye " : " Generate a brief summary of the paper# \n " ,
" title " : " 以《(.*)》为论文题目 " ,
" en-prompt " : " Write a brief summary of the English paper titled ‘ { $title}’, with a requirement of no more than { $wordnum} words "
} ,
" 生成课题的国内外研究状况综述# \n " : {
" data_tpye " : " A Summary of Research Status at Home and Abroad on Generated Topics# \n " ,
" title " : " 请写出以《(.*)》为课题的国内外研究状况综述 " ,
" en-prompt " : " Please provide an English summary of the research status of ‘ { $title}’ at home and abroad, with a word count of around { $wordnum} words "
} ,
" 生成6点本篇论文应完成的主要内容# \n " : {
" data_tpye " : " Generate 6 main contents that should be completed in this paper# \n " ,
" title " : " 请根据题目为《(.*)》 " ,
" opening_report_main_content " : " 研究内容为“(.*)”总结出至少6点本篇论文应完成的主要内容 " ,
" en-prompt " : " Please summarize at least 6 main English content that should be completed for this paper based on the title ‘ { $title}’ and the research content ‘ { $opening_report_main_content}’, using Arabic numerals for arrangement "
} ,
" 生成参考文献# \n " : {
" data_tpye " : " Generate references# \n " ,
" title " : " 论文题目是《(.*)》 " ,
" catalogue_str " : " 目录是“(.*)” " ,
" en-prompt " : " According to the paper title ‘ { $title}’ and the directory is ‘ { $catalogue_str}’, generate 15 references in the format of [1] xxx. "
} ,
" 生成论文小标题内容# \n " : {
" data_tpye " : " Generate paper subheading content# \n " ,
" title " : " 论文题目是《(.*)》 " ,
" catalogue_str " : " 目录是“(.*)” " ,
" smell-title " : " 请把其中的小标题“(.*?)”的内容补充完整 " ,
" en-prompt " : " According to the paper title ‘ { $title}’ and the table of contents ‘ { $catalogue_str}’, add approximately { $wordnum} words of English content to the subheading ‘ { $secondray_title}’. The content must include the current subheading and not include other titles in the table of contents "
} ,
" 生成论文摘要# \n " : {
" data_tpye " : " Generate paper abstract# \n " ,
" title " : " 论文题目是《(.*)》 " ,
" catalogue_str " : " 目录是“(.*)” " ,
" en-prompt " : " Generate a Chinese paper abstract based on the title “ { $title}” and the directory “ { $catalogue_str}”, with a required word count of around { $wordnum} words "
} ,
" 生成关键字# \n " : {
" data_tpye " : " Generate Keywords# \n " ,
" abstract " : " 请为“(.*)”这段论文摘要生成3-5个关键字 " ,
" en-prompt " : " Please generate 3-5 keywords for the abstract of the paper ‘ { $abstract}’ "
} ,
" 生成论文来源的背景-核心内容# \n " : {
" data_tpye " : " Background of generating paper sources - core content# \n " ,
" title " : " 以《(.*)》为论文题目 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Based on the paper title ‘ { $title}’ and the core content ‘ { $this->core_content}’, generate a background of approximately { $wordnum} words of English topic sources "
} ,
" 生成研究内容-核心内容# \n " : {
" data_tpye " : " Generate research content - core content# \n " ,
" title " : " 请帮我生成《(.*)》为题目 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Generate English research content based on the paper title ‘ { $title}’ and core content ‘ { $this->core_content}’, including an overall introduction and a summary of at least three aspects "
} ,
" 生成目录-核心内容# \n " : {
" data_tpye " : " Generate Directory - Core Content# \n " ,
" title " : " 为论文题目《(.*)》生成目录 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Generate an English directory based on the paper title ‘ { $title}’ and the core content ‘ { $this->core_content}’, with only the first and second level titles required. The format of the first level title is as follows: 1. xxx; The format of the secondary title is as follows: 1.1 xxx; 4 first level titles; Each first level title should contain at least 3 second level titles; "
} ,
" 生成课题的研究背景和意义-核心内容# \n " : {
" data_tpye " : " Research background and significance of the generated topic - core content# \n " ,
" title " : " 请分别写出以《(.*)》为课题 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Generate an English research background and significance of no less than { $wordnum} words based on the paper title ‘ { $title}’ and the core content ‘ { $this->core_content}’ "
} ,
" 生成论文简短总结-核心内容# \n " : {
" data_tpye " : " Generate a brief summary of the paper - core content# \n " ,
" title " : " 以《(.*)》为论文题目 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Generate a brief summary of the English paper with approximately { $wordnum} words based on the title ‘ { $title}’ and the core content ‘ { $this->core_content}’ "
} ,
" 生成课题的国内外研究状况综述-核心内容# \n " : {
" data_tpye " : " Overview of domestic and international research status on generating topics - core content# \n " ,
" title " : " 请写出以《(.*)》为课题 " ,
" hexin " : " 以“(.*)”为论文的研究方向 " ,
" en-prompt " : " Based on the paper title ‘ { $title}’ and the core content ‘ { $this->core_content}’, generate an English summary of research status at home and abroad with no less than { $wordnum} words "
}
}
with open ( " data/llama_t/chatglm_dev_4_prompt_llama.json " ) as f :
data = json . loads ( f . read ( ) )
data_new = [ ]
for i in tqdm ( data ) :
data_dan_dict = { }
instruction = i [ " instruction " ]
input_ = i [ " input " ]
input_ = str ( input_ ) . replace ( " \n " , " \\ n " )
try :
if instruction == " 生成论文来源的背景# \n " :
dan_dict = prompt_tpye_dict [ " 生成论文来源的背景# \n " ]
title = dan_dict [ " title " ]
title_re = re . findall ( title , input_ )
title = title_re [ 0 ]
en_title = predict_baichuan ( title )
data_dan_dict [ " en_title " ] = en_title
data_dan_dict [ " en_hexin " ] = " "
# data_dan_dict["output"] = en_output
data_new . append ( data_dan_dict )
#
# elif instruction == "生成研究内容#\n":
# dan_dict = prompt_tpye_dict["生成研究内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
#
# elif instruction == "生成目录#\n":
# dan_dict = prompt_tpye_dict["生成目录#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成课题的研究背景和意义#\n":
# dan_dict = prompt_tpye_dict["生成课题的研究背景和意义#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "1000")
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成致谢#\n":
# dan_dict = prompt_tpye_dict["生成致谢#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成论文简短总结#\n":
# dan_dict = prompt_tpye_dict["生成论文简短总结#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "300")
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成课题的国内外研究状况综述#\n":
# dan_dict = prompt_tpye_dict["生成课题的国内外研究状况综述#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# en_title = predict_baichuan(title)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "800")
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成6点本篇论文应完成的主要内容#\n":
# dan_dict = prompt_tpye_dict["生成6点本篇论文应完成的主要内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# opening_report_main_content = dan_dict["opening_report_main_content"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# opening_report_main_content_re = re.findall(opening_report_main_content, input_)
# opening_report_main_content = opening_report_main_content_re[0]
# opening_report_main_content = opening_report_main_content.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_opening_report_main_content = predict_baichuan(opening_report_main_content)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$opening_report_main_content}", en_opening_report_main_content)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成参考文献#\n":
# dan_dict = prompt_tpye_dict["生成参考文献#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# catalogue_str = dan_dict["catalogue_str"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# catalogue_str_re = re.findall(catalogue_str, input_)
# catalogue_str = catalogue_str_re[0]
# catalogue_str = catalogue_str.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_catalogue_str = predict_baichuan(catalogue_str)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$catalogue_str}", en_catalogue_str)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成论文小标题内容#\n":
# print(1)
# dan_dict = prompt_tpye_dict["生成论文小标题内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# catalogue_str = dan_dict["catalogue_str"]
# smell_title = dan_dict["smell-title"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# catalogue_str_re = re.findall(catalogue_str, input_)
# catalogue_str = catalogue_str_re[0]
# catalogue_str = catalogue_str.replace("\\n", "\n")
#
# smell_title_re = re.findall(smell_title, input_)
# print(smell_title_re)
# if smell_title_re == []:
# continue
# smell_title = smell_title_re[0]
# smell_title = smell_title.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_catalogue_str = predict_baichuan(catalogue_str)
# en_smell_title = predict_baichuan(smell_title)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$catalogue_str}", en_catalogue_str).replace("{$secondray_title}", en_smell_title).replace("{$wordnum}", "800")
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成论文摘要#\n":
# dan_dict = prompt_tpye_dict["生成论文摘要#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# catalogue_str = dan_dict["catalogue_str"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# catalogue_str_re = re.findall(catalogue_str, input_)
# catalogue_str = catalogue_str_re[0]
# catalogue_str = catalogue_str.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_catalogue_str = predict_baichuan(catalogue_str)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$catalogue_str}", en_catalogue_str).replace("{$wordnum}", "400")
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成关键字#\n":
# dan_dict = prompt_tpye_dict["生成关键字#\n"]
# en_prompt = dan_dict["en-prompt"]
# abstract = dan_dict["abstract"]
# catalogue_str = dan_dict["catalogue_str"]
# data_tpye = dan_dict["data_tpye"]
#
# abstract_re = re.findall(abstract, input_)
# abstract = abstract_re[0]
# abstract = abstract.replace("\\n", "\n")
#
# en_abstract = predict_baichuan(abstract)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$abstract}", en_abstract)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成论文来源的背景-核心内容#\n":
# dan_dict = prompt_tpye_dict["生成论文来源的背景-核心内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# hexin = dan_dict["hexin"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# hexin_re = re.findall(hexin, input_)
# hexin = hexin_re[0]
# hexin = hexin.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_hexin = predict_baichuan(hexin)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "200").replace("{$this->core_content}", en_hexin)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成研究内容-核心内容#\n":
# dan_dict = prompt_tpye_dict["生成研究内容-核心内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# hexin = dan_dict["hexin"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# hexin_re = re.findall(hexin, input_)
# hexin = hexin_re[0]
# hexin = hexin.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_hexin = predict_baichuan(hexin)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$this->core_content}", en_hexin)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成目录-核心内容#\n":
# dan_dict = prompt_tpye_dict["生成目录-核心内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# hexin = dan_dict["hexin"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# hexin_re = re.findall(hexin, input_)
# hexin = hexin_re[0]
# hexin = hexin.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_hexin = predict_baichuan(hexin)
# # en_output = predict_baichuan(i["output"])
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$this->core_content}", en_hexin)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成课题的研究背景和意义-核心内容#\n":
# dan_dict = prompt_tpye_dict["生成课题的研究背景和意义-核心内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# hexin = dan_dict["hexin"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# hexin_re = re.findall(hexin, input_)
# hexin = hexin_re[0]
# hexin = hexin.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_hexin = predict_baichuan(hexin)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "1000").replace("{$this->core_content}", en_hexin)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
#
# elif instruction == "生成论文简短总结-核心内容#\n":
# dan_dict = prompt_tpye_dict["生成论文简短总结-核心内容#\n"]
# en_prompt = dan_dict["en-prompt"]
# title = dan_dict["title"]
# hexin = dan_dict["hexin"]
# data_tpye = dan_dict["data_tpye"]
#
# title_re = re.findall(title, input_)
# title = title_re[0]
#
# hexin_re = re.findall(hexin, input_)
# hexin = hexin_re[0]
# hexin = hexin.replace("\\n", "\n")
#
# en_title = predict_baichuan(title)
# en_hexin = predict_baichuan(hexin)
# # en_output = predict_baichuan(i["output"])
# #
# # en_output_list = str(en_output).split(" ")
# # zishu = len(en_output_list)//100 * 100
#
# en_prompt = en_prompt.replace("{$title}", en_title).replace("{$wordnum}", "300").replace("{$this->core_content}", en_hexin)
#
# data_dan_dict["instruction"] = data_tpye
# data_dan_dict["input"] = en_prompt
# # data_dan_dict["output"] = en_output
#
# data_new.append(data_dan_dict)
if instruction == " 生成课题的国内外研究状况综述-核心内容# \n " :
dan_dict = prompt_tpye_dict [ " 生成课题的国内外研究状况综述-核心内容# \n " ]
title = dan_dict [ " title " ]
hexin = dan_dict [ " hexin " ]
title_re = re . findall ( title , input_ )
title = title_re [ 0 ]
hexin_re = re . findall ( hexin , input_ )
hexin = hexin_re [ 0 ]
hexin = hexin . replace ( " \\ n " , " \n " )
en_title = predict_baichuan ( title )
en_hexin = predict_baichuan ( hexin )
# en_output = predict_baichuan(i["output"])
#
# en_output_list = str(en_output).split(" ")
# zishu = len(en_output_list)//100 * 100
data_dan_dict [ " en_title " ] = en_title
data_dan_dict [ " en_hexin " ] = en_hexin
# data_dan_dict["output"] = en_output
data_new . append ( data_dan_dict )
except :
print ( i )
with open ( " ./data/llama_t/chatglm_en.json " , mode = " w " , encoding = " utf-8 " ) as f :
f . write ( json . dumps ( data_new , ensure_ascii = False , indent = 2 ) )
