@ -159,11 +159,11 @@ def get_dialogs_index(line: str):
return dialogs_text , dialogs_index , other_index
def chulichangju_1 ( text , snetence_id , chulipangban_return_list , short_num ) :
def chulichangju_1 ( text , snetence_id , chulipangban_return_list , short_num , sen_len ) :
fuhao = [ " ， " , " ？ " , " ！ " , " … " ]
dialogs_text , dialogs_index , other_index = get_dialogs_index ( text )
text_1 = text [ : 120 ]
text_2 = text [ 120 : ]
text_1 = text [ : sen_len ]
text_2 = text [ sen_len : ]
text_1_new = " "
if text_2 == " " :
chulipangban_return_list . append ( [ text_1 , snetence_id , short_num ] )
@ -176,7 +176,7 @@ def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num):
text_1_new + = text_1 [ i ]
chulipangban_return_list . append ( [ text_1_new , snetence_id , short_num ] )
if text_2 != " " :
if i + 1 != 120 :
if i + 1 != sen_len :
text_2 = text_1 [ i + 1 : ] + text_2
break
# else:
@ -185,7 +185,7 @@ def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num):
chulipangban_return_list . append ( [ text_1 , snetence_id , short_num ] )
if text_2 != " " :
short_num + = 1
chulipangban_return_list = chulichangju_1 ( text_2 , snetence_id , chulipangban_return_list , short_num )
chulipangban_return_list = chulichangju_1 ( text_2 , snetence_id , chulipangban_return_list , short_num , sen_len )
return chulipangban_return_list
@ -289,11 +289,11 @@ def get_multiple_urls(text_info):
if text_info [ i ] [ 3 ] == True :
input_values . append ( text_info [ i ] [ 4 ] )
input_index . append ( i )
with concurrent . futures . ThreadPoolExecutor ( 100 ) as executor :
with concurrent . futures . ThreadPoolExecutor ( 64 ) as executor :
# 使用map方法并发地调用worker_function
results_1 = list ( executor . map ( request_api_chatgpt , input_values ) )
with concurrent . futures . ThreadPoolExecutor ( 100 ) as executor :
with concurrent . futures . ThreadPoolExecutor ( 64 ) as executor :
# 使用map方法并发地调用worker_function
results = list ( executor . map ( uuid_search_mp , [ results_1 ] ) )
@ -312,10 +312,9 @@ def get_multiple_urls(text_info):
return text_info
def chulipangban_test_1 ( snetence_id , text ) :
def chulipangban_test_1 ( snetence_id , text , sen_len ) :
# 引号处理
dialogs_text , dialogs_index , other_index = get_dialogs_index ( text )
for dialogs_text_dan in dialogs_text :
text_dan_list = text . split ( dialogs_text_dan )
@ -329,7 +328,7 @@ def chulipangban_test_1(snetence_id, text):
is_chinese = False
else :
spilt_word = " 。 "
spilt_sen_len = 120
spilt_sen_len = sen_len
is_chinese = True
# 存放整理完的数据
@ -358,7 +357,7 @@ def chulipangban_test_1(snetence_id, text):
# sentence_pre = autotitle.gen_synonyms_short(sentence)
# return_list.append(sentence_pre)
else :
sentence_split_list = chulichangju_1 ( sentence , snetence_id , [ ] , 0 )
sentence_split_list = chulichangju_1 ( sentence , snetence_id , [ ] , 0 , sen_len )
for sentence_short in sentence_split_list [ : - 1 ] :
sentence_batch_list . append ( sentence_short )
sentence_split_list [ - 1 ] [ 0 ] = sentence_split_list [ - 1 ] [ 0 ] + spilt_word
@ -371,7 +370,7 @@ def chulipangban_test_1(snetence_id, text):
# sentence_pre = autotitle.gen_synonyms_short(sentence)
# return_list.append(sentence_pre)
else :
sentence_split_list = chulichangju_1 ( sentence_list [ - 1 ] , snetence_id , [ ] , 0 )
sentence_split_list = chulichangju_1 ( sentence_list [ - 1 ] , snetence_id , [ ] , 0 , sen_len )
for sentence_short in sentence_split_list :
sentence_batch_list . append ( sentence_short )
@ -381,8 +380,13 @@ def chulipangban_test_1(snetence_id, text):
def paragraph_test ( texts : dict ) :
text_new = [ ]
for i , text in texts . items ( ) :
text_list = chulipangban_test_1 ( i , text )
text_new . extend ( text_list )
bool_ = has_chinese ( text )
if bool_ == True :
text_list = chulipangban_test_1 ( i , text , sen_len = 120 )
text_new . extend ( text_list )
else :
text_list = chulipangban_test_1 ( i , text , sen_len = 500 )
text_new . extend ( text_list )
# text_new_str = "".join(text_new)
return text_new
@ -508,12 +512,12 @@ def english_ulit(sentence):
# 判断句子长度
if sentence != " " :
if sentence [ - 1 ] != " . " :
text = f " <|role|>user<|says|> Rewrite the following sentence fragment, ensuring that the meaning remains similar but with significant changes. The length of the rewritten sentence must be greater, not less. Additionally, the words in the short sentences must connect seamlessly with the preceding and following sentences: \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
else :
text = f " <|role|>user<|says|> Rewrite the following sentence, requiring the meaning to be similar but the change to be larger, and the number of words can only be more but not less: \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
else :
if_change = False
text = f " <|role|>user<|says|> The following words remain unchanged \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
return text , if_change
@ -526,12 +530,12 @@ def chinese_ulit(sentence):
if len ( sentence ) > 9 :
if sentence [ - 1 ] != " 。 " :
text = f " <|role|>user<|says|> 任务：改写句子 \n 改写下面半这句话，要求意思接近但是改动幅度比较大，字数只能多不能少，短句前后词跟上下句衔接不能有错误: \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n 改写下面半这句话，要求意思接近但是改动幅度比较大，字数只能多不能少，短句前后词跟上下句衔接不能有错误: \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
else :
text = f " <|role|>user<|says|> 任务：改写句子 \n 改写下面这句话，要求意思接近但是改动幅度比较大，字数只能多不能少: \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n 改写下面这句话，要求意思接近但是改动幅度比较大，字数只能多不能少: \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
else :
text = f " <|role|>user<|says|> 下面词不做任何变化: \n { sentence } <|end|> \n <|role|>assistant<|says|> "
text = f " <|im_start|>user \n \n { sentence } <|im_ end|> \n <|im_start|>assistant \n "
if_change = False
return text , if_change
majiahui@haimaqingfan.com
2 months ago