diff --git a/flask_drop_rewrite_request.py b/flask_drop_rewrite_request.py index 17936d5..5a9ba67 100644 --- a/flask_drop_rewrite_request.py +++ b/flask_drop_rewrite_request.py @@ -159,11 +159,11 @@ def get_dialogs_index(line: str): return dialogs_text, dialogs_index, other_index -def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num): +def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num, sen_len): fuhao = [",", "?", "!", "…"] dialogs_text, dialogs_index, other_index = get_dialogs_index(text) - text_1 = text[:120] - text_2 = text[120:] + text_1 = text[:sen_len] + text_2 = text[sen_len:] text_1_new = "" if text_2 == "": chulipangban_return_list.append([text_1, snetence_id, short_num]) @@ -176,7 +176,7 @@ def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num): text_1_new += text_1[i] chulipangban_return_list.append([text_1_new, snetence_id, short_num]) if text_2 != "": - if i + 1 != 120: + if i + 1 != sen_len: text_2 = text_1[i + 1:] + text_2 break # else: @@ -185,7 +185,7 @@ def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num): chulipangban_return_list.append([text_1, snetence_id, short_num]) if text_2 != "": short_num += 1 - chulipangban_return_list = chulichangju_1(text_2, snetence_id, chulipangban_return_list, short_num) + chulipangban_return_list = chulichangju_1(text_2, snetence_id, chulipangban_return_list, short_num, sen_len) return chulipangban_return_list @@ -289,11 +289,11 @@ def get_multiple_urls(text_info): if text_info[i][3] == True: input_values.append(text_info[i][4]) input_index.append(i) - with concurrent.futures.ThreadPoolExecutor(100) as executor: + with concurrent.futures.ThreadPoolExecutor(64) as executor: # 使用map方法并发地调用worker_function results_1 = list(executor.map(request_api_chatgpt, input_values)) - with concurrent.futures.ThreadPoolExecutor(100) as executor: + with concurrent.futures.ThreadPoolExecutor(64) as executor: # 使用map方法并发地调用worker_function results = list(executor.map(uuid_search_mp, [results_1])) @@ -312,10 +312,9 @@ def get_multiple_urls(text_info): return text_info -def chulipangban_test_1(snetence_id, text): +def chulipangban_test_1(snetence_id, text, sen_len): # 引号处理 - dialogs_text, dialogs_index, other_index = get_dialogs_index(text) for dialogs_text_dan in dialogs_text: text_dan_list = text.split(dialogs_text_dan) @@ -329,7 +328,7 @@ def chulipangban_test_1(snetence_id, text): is_chinese = False else: spilt_word = "。" - spilt_sen_len = 120 + spilt_sen_len = sen_len is_chinese = True # 存放整理完的数据 @@ -358,7 +357,7 @@ def chulipangban_test_1(snetence_id, text): # sentence_pre = autotitle.gen_synonyms_short(sentence) # return_list.append(sentence_pre) else: - sentence_split_list = chulichangju_1(sentence, snetence_id, [], 0) + sentence_split_list = chulichangju_1(sentence, snetence_id, [], 0, sen_len) for sentence_short in sentence_split_list[:-1]: sentence_batch_list.append(sentence_short) sentence_split_list[-1][0] = sentence_split_list[-1][0] + spilt_word @@ -371,7 +370,7 @@ def chulipangban_test_1(snetence_id, text): # sentence_pre = autotitle.gen_synonyms_short(sentence) # return_list.append(sentence_pre) else: - sentence_split_list = chulichangju_1(sentence_list[-1], snetence_id, [], 0) + sentence_split_list = chulichangju_1(sentence_list[-1], snetence_id, [], 0, sen_len) for sentence_short in sentence_split_list: sentence_batch_list.append(sentence_short) @@ -381,8 +380,13 @@ def chulipangban_test_1(snetence_id, text): def paragraph_test(texts: dict): text_new = [] for i, text in texts.items(): - text_list = chulipangban_test_1(i, text) - text_new.extend(text_list) + bool_ = has_chinese(text) + if bool_ == True: + text_list = chulipangban_test_1(i, text, sen_len=120) + text_new.extend(text_list) + else: + text_list = chulipangban_test_1(i, text, sen_len=500) + text_new.extend(text_list) # text_new_str = "".join(text_new) return text_new @@ -508,12 +512,12 @@ def english_ulit(sentence): # 判断句子长度 if sentence != "": if sentence[-1] != ".": - text = f"<|role|>user<|says|>Rewrite the following sentence fragment, ensuring that the meaning remains similar but with significant changes. The length of the rewritten sentence must be greater, not less. Additionally, the words in the short sentences must connect seamlessly with the preceding and following sentences:\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\nRewrite the following sentence fragment, ensuring that the meaning remains similar but with significant changes. The length of the rewritten sentence must be greater, not less. Additionally, the words in the short sentences must connect seamlessly with the preceding and following sentences:\n{sentence}<|im_end|>\n<|im_start|>assistant\n" else: - text = f"<|role|>user<|says|>Rewrite the following sentence, requiring the meaning to be similar but the change to be larger, and the number of words can only be more but not less:\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\nRewrite the following sentence, requiring the meaning to be similar but the change to be larger, and the number of words can only be more but not less:\n{sentence}<|im_end|>\n<|im_start|>assistant\n" else: if_change = False - text = f"<|role|>user<|says|>The following words remain unchanged\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\nThe following words remain unchanged\n{sentence}<|im_end|>\n<|im_start|>assistant\n" return text, if_change @@ -526,12 +530,12 @@ def chinese_ulit(sentence): if len(sentence) > 9: if sentence[-1] != "。": - text = f"<|role|>user<|says|>任务:改写句子\n改写下面半这句话,要求意思接近但是改动幅度比较大,字数只能多不能少,短句前后词跟上下句衔接不能有错误:\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\n任务:改写句子\n改写下面半这句话,要求意思接近但是改动幅度比较大,字数只能多不能少,短句前后词跟上下句衔接不能有错误:\n{sentence}<|im_end|>\n<|im_start|>assistant\n" else: - text = f"<|role|>user<|says|>任务:改写句子\n改写下面这句话,要求意思接近但是改动幅度比较大,字数只能多不能少:\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\n任务:改写句子\n改写下面这句话,要求意思接近但是改动幅度比较大,字数只能多不能少:\n{sentence}<|im_end|>\n<|im_start|>assistant\n" else: - text = f"<|role|>user<|says|>下面词不做任何变化:\n{sentence}<|end|>\n<|role|>assistant<|says|>" + text = f"<|im_start|>user\n下面词不做任何变化:\n{sentence}<|im_end|>\n<|im_start|>assistant\n" if_change = False return text, if_change