mulu_ner/题目去重.py

title_list = []


with open("./data/题目3.txt", encoding="utf-8") as f:
    text = f.read()

text_list = text.split("\n")


for i in text_list:
    title_list.append(i.split("@@@@@")[0])

with open("./data/题目2.txt", encoding="utf-8") as f:
    text = f.read()

text_list = text.split("\n")

for i in text_list:
    title_list.append(i.split("@@@@@")[0])

print(title_list)

with open("./data/题目4.txt", encoding="utf-8") as f:
    text = f.read()

text_list = text.split("\n")


title_list_new = []
for i in text_list:
    if i.split("@@@@@")[0] in title_list:
        continue
    else:
        title_list_new.append(i)

print(len(title_list_new))

with open("./data/题目4_new.txt", mode="w",encoding="utf-8") as f:
    for i in title_list_new:
        f.write(i)
        f.write("\n")