Browse Source

remove dummy code

main
hiyouga 2 years ago
parent
commit
a72492e649
  1. 2
      data/dataset_info.json
  2. 3
      src/utils/common.py

2
data/dataset_info.json

@ -98,7 +98,7 @@
"file_name": "wiki_demo.txt",
"file_sha1": "b2288edb05b233e5b35250fd4b308a5fa21fa66d",
"columns": {
"prompt": "instruction",
"prompt": "text",
"query": "",
"response": "",
"history": ""

3
src/utils/common.py

@ -316,9 +316,6 @@ def prepare_data(
max_samples_temp = min(len(dataset), max_samples)
dataset = dataset.select(range(max_samples_temp))
if dataset.column_names[0] == "text": # for plaintext (in pre-training)
dataset = dataset.rename_column("text", getattr(dataset_attr, "prompt_column"))
dummy_data = [None] * len(dataset)
for column_name, target_name in [
("prompt_column", "prompt"),

Loading…
Cancel
Save