mulu_ner/chatgpt数据清洗整合训练数据.py

import os
import json
import re
import math
import numpy as np
from tqdm import tqdm
import random

path_output = "chatgpt_data_v1"
patten = "目录是“(.*)”，请把其中的"
pantten_biaoti = '[1-9一二三四五六七八九ⅠⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5a-zA-Z]+'
thanks = "致谢"
references = "参考文献"
excursus = "附录"
RE_CHINA_NUMS = "[一二三四五六七八九].?.?总结|[1-9].?.?总结|[一二三四五六七八九].?.?结论|[1-9].?.?结论"
RE_CHINA_TITLE = "请把其中的小标题“(.*?)”的内容补充完整|请把其中的大标题“(.*?)”的内容补充完整"

data_tongji = {
    "0-600": 0,
    "600-1500": 0,
    "1500-": 0,
}

jishu = 0

def is_contains_chinese(strs):
    for _char in strs:
        if '\u4e00' <= _char <= '\u9fa5':
            return True
    return False
# pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+'

lable_data_amount = {
    "title_beijing_prompt_data.txt": {"num_token": -1, "prompt": "生成论文来源的背景#"},
    "title_jianjie_prompt_data.txt": {"num_token": -1, "prompt": "生成研究内容#"},
    "title_mulu_prompt_data.txt": {"num_token": -1, "prompt": "生成目录#"},
    "title_yanjiubeijingyiyi_prompt_data.txt": {"num_token": -1, "prompt": "生成课题的研究背景和意义#"},
    "title_zhixie_prompt_data.txt": {"num_token": -1, "prompt": "生成致谢#"},
    "title_zongjie_prompt_data.txt": {"num_token": -1, "prompt": "生成论文简短总结#"},
    "title_zongshu_prompt_data.txt": {"num_token": -1, "prompt": "生成课题的国内外研究状况综述#"},
    "sencend_task_book_prompt_data.txt": {"num_token": -1, "prompt": "生成6点本篇论文应完成的主要内容#"},
    "sencend_references_prompt_data.txt": {"num_token": -1, "prompt": "生成参考文献#"},
    "sencend_small_title_prompt_shuffle_data.txt": {"num_token": -1, "prompt": "生成论文小标题内容#"},
    "sencend_zhaiyao_prompt_data.txt": {"num_token": -1, "prompt": "生成论文摘要#"},
    "third_zhaiyao_chinese_keyword_prompt_data.txt": {"num_token": -1, "prompt": "生成关键字#"},
    "third_zhaiyao_fanyi_prompt_data.txt": {"num_token": -1, "prompt": "翻译摘要#"},
    "fourth_chinese_keyword_en_prompt_data.txt": {"num_token": -1, "prompt": "翻译关键词#"},
    "title_hexin_beijing_prompt_data.txt": {"num_token": -1, "prompt": "生成论文来源的背景#"},
    "title_hexin_jianjie_prompt_data.txt": {"num_token": -1, "prompt": "生成研究内容#"},
    "title_hexin_mulu_prompt_data.txt": {"num_token": -1, "prompt": "生成目录#"},
    "title_hexin_yanjiubeijingyiyi_prompt_data.txt": {"num_token": -1, "prompt": "生成课题的研究背景和意义#"},
    "title_hexin_zongjie_prompt_data.txt": {"num_token": -1, "prompt": "生成论文简短总结#"},
    "title_hexin_zongshu_prompt_data.txt": {"num_token": -1, "prompt": "生成课题的国内外研究状况综述#"},
    "title_hexin_zhixie_prompt_data.txt": {"num_token": -1, "prompt": "生成致谢#"}
}


patten_mulu = {
    "title_mulu_references_prompt_data.txt": "目录是“(.*)”，请为这篇论文生成15篇左右的参考文献",
    "title_mulu_small_title_prompt_shuffle_data_new.txt": "目录是“(.*)”，请把其中的小标题",
    "title_mulu_zhaiyao_data.txt": "目录是“(.*)”，生成论文摘要"
}



path_list = []
file = "./data/{}/paper_prompt_title_1".format(path_output)
for root, dirs, files in os.walk(file):
    for file in files:
        path = os.path.join(root, file)
        path_list.append(path)

file = "./data/{}/paper_prompt_title_1_1".format(path_output)
for root, dirs, files in os.walk(file):
    for file in files:
        path = os.path.join(root, file)
        path_list.append(path)

file = "./data/{}/paper_prompt_title_1_1_1".format(path_output)
for root, dirs, files in os.walk(file):
    for file in files:
        path = os.path.join(root, file)
        path_list.append(path)

file = "./data/{}/paper_prompt_title_1_1_1_1".format(path_output)
for root, dirs, files in os.walk(file):
    for file in files:
        path = os.path.join(root, file)
        path_list.append(path)

file = "./data/{}/paper_prompt_title_1_hexin".format(path_output)
for root, dirs, files in os.walk(file):
    for file in files:
        path = os.path.join(root, file)
        path_list.append(path)

text_list_new = []
tongji = {}

if __name__ == '__main__':

    data_list = []
    train_list = []
    new_data_list = []
    for path in path_list:
        patten_mulu_bool = False
        shuminghao_bool = False
        patten_mulu_patten = ""
        task_name = path.split("\\")[-1]

        if task_name in patten_mulu:
            patten_mulu_bool = True
            patten_mulu_patten = patten_mulu[task_name]

        train_data_amount_dict = lable_data_amount[task_name]
        train_data_amount = train_data_amount_dict["num_token"]
        prompt = train_data_amount_dict["prompt"]

        with open(path, encoding="utf-8") as f:
            text = f.read()
            text_list = text.split("@" * 20)

        for data_dan in text_list:
            if "*" * 20 in data_dan:
                content, summary = data_dan.split("*" * 20)
                # text_list_new.append(data_dan)
                # data_dan = data_dan.replace("\\n", "\n").replace("\n", "\\n")

                new_data_list.append(
                    {
                        "input": str(content).strip("\"").strip("\n").strip("\"").replace("\\n", "\n"),
                        "output": str(summary).replace("\\n", "\n").strip("\""),
                        "instruction": prompt
                    }
                )

                if task_name not in tongji:
                    tongji[task_name] = 1
                else:
                    tongji[task_name] += 1
            else:
                continue

    for data_dan in tqdm(new_data_list):
        zishu_query = len(data_dan["input"])
        zishu_response = len(data_dan["output"])

        prompt = str(data_dan["instruction"]).replace("\\n", "\n")
        query = data_dan["input"].replace("\\n", "\n")
        response = data_dan["output"].replace("\\n", "\n")

        if prompt == "翻译摘要#":
            zishu_summary = len(response.split(" "))
        elif prompt == "翻译关键词#":
            zishu_summary = len(response.split(" "))
        else:
            bool_ = is_contains_chinese(response)
            if bool_ == False:
                print(data_dan)
                continue

        if "生成方向" in query:
            query = query.replace("生成方向","研究方向")
        if "生成方向" in response:
            response = response.replace("生成方向", "研究方向")

        if prompt == "生成论文小标题内容#":
            query_re = re.findall(RE_CHINA_TITLE, query)
            if "总结" not in query_re[0] or "结论" not in query_re[0]:
                response_re = re.findall(RE_CHINA_NUMS, response)
                if response_re != []:
                    jishu += 1
                    continue

        if prompt[-1] != "\n":
            prompt += "\n"
        if query[-1] != "\n":
            query += "\n"
        query = "问：" + query + "答：\n"

        if len(query) < 700 and len(response) < 1400:
            data_list.append({
                "instruction": prompt,
                "input": query,
                "output": response
            })

    train_nums = int(len(data_list) * 0.95)
    dev_nums = int(len(data_list) * 0.05)
    #
    random.shuffle(data_list)
    print(train_nums)
    print(dev_nums)
    train_list = data_list[:train_nums]
    dev_list = data_list[train_nums:]
    with open("./data/llama_t/chatglm_train_4_prompt_llama.json", mode="w", encoding="utf-8") as f:
        f.write(json.dumps(train_list, ensure_ascii=False, indent=2))

    with open("./data/llama_t/chatglm_dev_4_prompt_llama.json", mode="w", encoding="utf-8") as f:
        f.write(json.dumps(dev_list, ensure_ascii=False, indent=2))