fix bug in freeze tuning

Former-commit-id: f6b436a08421ca17d64abc51497f4aa43729a43b
2023-11-16 14:25:11 +08:00
parent e59a3d71e0
commit f9d4e37b3c
2 changed files with 8 additions and 1 deletions
--- a/src/llmtuner/train/utils.py
+++ b/src/llmtuner/train/utils.py
@@ -76,4 +76,5 @@ def create_reward_model(
        reward_finetuning_args = FinetuningArguments(finetuning_type="lora")
        reward_model, _ = load_model_and_tokenizer(reward_model_args, reward_finetuning_args, is_trainable=False, stage="ppo")
        logger.info("Load full weights of reward model from {}".format(finetuning_args.reward_model))
+        logger.warning("Please ensure the ppo model and reward model share SAME tokenizer and vocabulary.")
        return reward_model