训练文本生成
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

24 lines
789 B

2 years ago
# coding=utf-8
# Exports the fine-tuned model.
2 years ago
# Usage: python export_model.py --checkpoint_dir path_to_checkpoint --output_dir path_to_save_model
from transformers import HfArgumentParser, TrainingArguments
2 years ago
from utils import ModelArguments, FinetuningArguments, load_pretrained
2 years ago
def main():
2 years ago
parser = HfArgumentParser((ModelArguments, TrainingArguments, FinetuningArguments))
model_args, training_args, finetuning_args = parser.parse_args_into_dataclasses()
2 years ago
2 years ago
model, tokenizer = load_pretrained(model_args, finetuning_args)
2 years ago
model.save_pretrained(training_args.output_dir, max_shard_size="1GB")
tokenizer.save_pretrained(training_args.output_dir)
print("model and tokenizer have been saved at:", training_args.output_dir)
if __name__ == "__main__":
main()