jiahui
/
crawling_directory_filtering
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

目录筛选升级

master
majiahui@haimaqingfan.com 1 year ago
parent
89b5f2a04f
commit
3c411c7948
4 changed files with 81 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .idea/爬取目录筛选.iml
  2. 3
      数据合并.py
  3. 30
      筛选10000条目录.py
  4. 70
      读取结果生成目录.py

2
.idea/爬取目录筛选.iml
View File

@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.8 (ldm)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Remote Python 3.9.0 (sftp://majiahui@192.168.31.74:22/home/majiahui/miniconda3/envs/llama_paper/bin/python)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

3
数据合并.py
View File

@ -1,3 +1,4 @@
import json
# json.load()
@ -17,7 +18,7 @@ for chunk in pd.read_csv(filename, chunksize=chunksize):
print(1)
# 对每个 chunk 进行处理
# print(chunk.columns)
print(chunk.columns)
df_list = chunk.values.tolist()
# print(df_list[0])
for i in range(len(df_list)):

30
筛选10000条目录.py
View File

@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
import json
with open("mulu_prompt.json") as f:
with open("data/mulu_prompt_1.json", encoding="utf-8") as f:
mulu_list = json.loads(f.read())
import random
@ -10,20 +11,41 @@ random.shuffle(mulu_list)
random.shuffle(mulu_list)
random.shuffle(mulu_list)
mulu_list_new = mulu_list[:10000]
mulu_list_new = mulu_list[:30000]
# {
# "instruction": "任务：生成论文小标题内容",
# "input": "背景:我是一名博士生，我想写一篇论文。\n角色：我需要你作为拥有google学术论文数据库和百度学术论文数据库的资深数据。\n要求：根据论文题目“大数据时代的数据分析与挖掘”和目录“一、大数据时代概述\n1.1 大数据的定义与特征\n1.2 大数据时代的发展历程\n1.3 大数据时代的挑战与机遇\n二、数据分析与挖掘的基本概念\n2.1 数据分析的概念与目的\n2.2 数据挖掘的概念与任务\n2.3 数据分析与数据挖掘的关系\n三、大数据时代的数据分析技术\n3.1 分布式计算技术\n3.2 数据可视化技术\n3.3 机器学习与深度学习技术\n四、大数据时代的挖掘技术\n4.1 关联规则挖掘\n4.2 聚类分析\n4.3 时间序列分析\n五、数据挖掘的应用与实践\n5.1 金融行业的数据挖掘应用\n5.2 电商行业的数据挖掘应用\n5.3 医疗行业的数据挖掘应用”，为小标题“1.1 大数据的定义与特征”填充400字左右的中文内容，内容第一行返回“1.1 大数据的定义与特征”且不包含目录中其他标题，禁止出现“首先”，“其次”，等字样，必要时候应根据条数罗列",
# "output": "1.1 大数据的定义与特征\n\n大数据是指传统数据处理应用软件难以处理的大规模和复杂度的数据集合。IDC 的定义为：\"大数据技术指的是那些使用新型处理模式，以经济有效的方式，从高速增长的各类数据中提炼出价值的技术与概念。\"大数据的特征通常用“5V”来概述，即体量巨大（Volume）、速度快（Velocity）、种类多（Variety）、真实性（Veracity）和价值密度（Value）。\n\n(1) 体量巨大：数据量巨大，处于TB（太字节）到PB（拍字节）甚至更高的量级。\n\n(2) 速度快：数据流入的速度极快，实时性成为数据处理的关键要求。\n\n(3) 种类多：数据类型繁多，包括结构化数据、非结构化数据和半结构化数据。\n\n(4) 真实性：数据的质量和准确性会直接影响到分析结果的可行性。\n\n(5) 价值密度：大数据中蕴含有巨大的价值，但相比数据量的庞大，这些价值的密度却相对较低，需要通过有效的分析才能挖掘出来。\n\n因此，面对大数据的定义与特征，数据分析与挖掘的技术与方法必须适应数据的快速流转和多样性，且须具备处理海量数据的能力，以期从中获取有价值的信息与知识。"
# },
shaixuan = {
"一级标题生成2个": 0,
"一级标题生成3个": 0,
"一级标题生成4个": 0,
"一级标题生成5个": 0,
"一级标题生成6个": 0,
"一级标题生成7个": 0,
"一级标题生成8个": 0,
"一级标题生成9个": 0,
}
mulu_list_json = []
for i in mulu_list_new:
mulu_list_json.append({
"instruction": "任务：生成目录",
"input": i[0].replace("一级标题不少于7个", "一级标题5-7个"),
"input": i[0],
"output": i[1]
})
with open("mulu_prompt_10000.json", "w", encoding="utf-8") as f:
for geshu_str in shaixuan:
if geshu_str in i[0]:
shaixuan[geshu_str] += 1
print(shaixuan)
with open("mulu_prompt_30000_1.json", "w", encoding="utf-8") as f:
f.write(json.dumps(mulu_list_json, ensure_ascii=False, indent=2))

70
读取结果生成目录.py
View File

@ -5,13 +5,17 @@ import re
pantten_dabiaoti = '^第([0-9一二三四五六七八九十]{1,})?章\s{1,}?(.*)|^([0-9一二三四五六七八九十]{1,}?)\s{1,}?(.*)'
pantten_xiaobiaoti = '^[0-9](\.[0-9]\d*){1,3}\s{1,}?.*$'
pantten_dabiaoti_shai = '^([5-7五六七])、(.*)'
pantten_xiaobiaoti_shai = '^[5-7](\.[5-7]){1,2}\s{1,}?.*$'
pantten_dabiaoti_shai = '^([3-9三四五六七八九])、(.*)'
pantten_xiaobiaoti_shai = '^[3-9](\.[3-9]){1,2}\s{1,}?.*$'
pantten_dabiaoti_ = '^([八九])、(.*)'
pantten_xiaobiaoti_1 = '^[1-7](\.[9]){1,2}\s{1,}?.*$'
pantten_xiaobiaoti_2 = '^[1-7](\.[1-9]{2,}?){1,2}\s{1,}?.*$'
pantten_xiaobiaoti_3 = '^[1-7](\.[1-8]){3,}\s{1,}?.*$'
pantten_xiaobiaoti_1 = '^[1-9](\.[9]){1,2}\s{1,}?.*$'
pantten_xiaobiaoti_2 = '^[1-9](\.[1-9]{2,}?){1,2}\s{1,}?.*$'
pantten_xiaobiaoti_3 = '^[1-9](\.[1-8]){3,}\s{1,}?.*$'
pantten_dabiaoti_geshu = '^([1-9一二三四五六七八九])、(.*)'
pantten_xiaobiaoti_geshu = '^[1-9](\.[1-9]){1}\s{1,}?.*$'
pantten_weijinci_list = ["论文结构"]
def contains_chinese(text):
# 检查是否包含中文字符
@ -59,13 +63,19 @@ def mulu_ulit(mulu):
continue
if return_bool == True:
for weijinci in pantten_weijinci_list:
for i in mulu_new:
if weijinci in i:
return_bool = False
break
if return_bool == True:
for i in mulu_new:
dabiaoti = re.findall(pantten_dabiaoti_, i)
xiaobiaoti_1 = re.findall(pantten_xiaobiaoti_1, i)
xiaobiaoti_2 = re.findall(pantten_xiaobiaoti_2, i)
xiaobiaoti_3 = re.findall(pantten_xiaobiaoti_3, i)
if list(set(dabiaoti)| set(xiaobiaoti_1)| set(xiaobiaoti_2)| set(xiaobiaoti_3)) != []:
if list(set(xiaobiaoti_1)| set(xiaobiaoti_2)| set(xiaobiaoti_3)) != []:
return_bool = False
break
@ -94,32 +104,56 @@ def mulu_ulit(mulu):
if chinese_bool == False:
return_bool = False
return [return_bool, mulu_new]
dabiaoti_geshu_jishu = 0
xiaobiaoti_min = 0
xiaobiaoti_max = 0
if return_bool == True:
dabiaoti_geshu_jishu = 0
xiaobiaoti_geshu_jishu_list = []
for i in mulu_new:
dabiaoti_geshu = re.findall(pantten_dabiaoti_geshu, i)
if dabiaoti_geshu != []:
dabiaoti_geshu_jishu += 1
xiaobiaoti_geshu_jishu_list.append(0)
xiaobiaoti_geshu = re.findall(pantten_xiaobiaoti_geshu, i)
if xiaobiaoti_geshu != []:
xiaobiaoti_geshu_jishu_list[-1] += 1
xiaobiaoti_min = min(xiaobiaoti_geshu_jishu_list)
xiaobiaoti_max = max(xiaobiaoti_geshu_jishu_list)
return [return_bool, mulu_new, dabiaoti_geshu_jishu, xiaobiaoti_min, xiaobiaoti_max]
prompt = "为论文题目“{}”生成中文目录，要求只有一级标题，二级标题和三级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；三级标题使用阿拉伯数字 例如1.1.2 xxx；一级标题不少于7个；每个一级标题至少包含3个二级标题；三级标题个数不限制"
prompt_min_max = "为论文题目“{}”生成中文目录，要求只有一级标题，二级标题和三级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；三级标题使用阿拉伯数字 例如1.1.2 xxx；一级标题生成{}个；每个一级标题包含{}-{}个二级标题；三级标题个数不限制"
prompt_not_min_max = "为论文题目“{}”生成中文目录，要求只有一级标题，二级标题和三级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；三级标题使用阿拉伯数字 例如1.1.2 xxx；一级标题生成{}个；每个一级标题包含{}个二级标题；三级标题个数不限制"
mulu_zong = []
def shengcheng_prompt(title, mulu_list):
mulu_prompt = prompt.format(title)
mulu = "\n".join(mulu_list)
def shengcheng_prompt(title, mulu_list, dabiaoti_geshu_jishu, xiaobiaoti_min, xiaobiaoti_max):
if xiaobiaoti_min != xiaobiaoti_max:
mulu_prompt = prompt_min_max.format(title, dabiaoti_geshu_jishu, xiaobiaoti_min, xiaobiaoti_max)
mulu = "\n".join(mulu_list)
else:
mulu_prompt = prompt_not_min_max.format(title, dabiaoti_geshu_jishu, xiaobiaoti_min)
mulu = "\n".join(mulu_list)
return mulu_prompt, mulu
with open("data.json", encoding="utf-8") as f:
with open("data/data.json", encoding="utf-8") as f:
for i in f.readlines():
a = json.loads(i)
try:
return_bool, mulu_new = mulu_ulit(json.loads(a[-1]))
return_bool, mulu_new, dabiaoti_geshu_jishu, xiaobiaoti_min, xiaobiaoti_max= mulu_ulit(json.loads(a[-1]))
if return_bool == True:
mulu_zong.append(shengcheng_prompt(a[0], mulu_new))
mulu_zong.append(shengcheng_prompt(a[0], mulu_new, dabiaoti_geshu_jishu, xiaobiaoti_min, xiaobiaoti_max))
# else:
# print("===========================================================================")
# print(mulu_new)
except:
continue
print(len(mulu_zong))
with open("mulu_prompt.json", "w", encoding="utf-8") as f:
f.write(json.dumps(mulu_zong, indent=2))
with open("data/mulu_prompt_1.json", "w", encoding="utf-8") as f:
f.write(json.dumps(mulu_zong, ensure_ascii=False, indent=2))
Write Preview
Loading…
Cancel
Save