change to right-padding, update reward score #803

Former-commit-id: baa90415bc8f5ebd423d001378b51c3a3a6c2ec7

src/llmtuner/tuner/sft/trainer.py

@@ -50,10 +50,9 @@ class Seq2SeqPeftTrainer(PeftTrainer):
         loss, generated_tokens, labels = super().prediction_step(
             model, inputs, prediction_loss_only=prediction_loss_only, ignore_keys=ignore_keys
         )
-        if generated_tokens is not None:
-            generated_tokens[:, :max(prompt_len, label_len)] = (
-                self.tokenizer.pad_token_id * torch.ones_like(generated_tokens[:, :max(prompt_len, label_len)])
-            )
+        generated_tokens = (
+            generated_tokens[:, max(prompt_len, label_len):] if generated_tokens is not None else None
+        )
 
         return loss, generated_tokens, labels
 

src/llmtuner/tuner/sft/workflow.py

@@ -27,6 +27,10 @@ def run_sft(
     dataset = get_dataset(model_args, data_args)
     model, tokenizer = load_model_and_tokenizer(model_args, finetuning_args, training_args.do_train, stage="sft")
     dataset = preprocess_dataset(dataset, tokenizer, data_args, training_args, stage="sft")
+
+    if training_args.predict_with_generate:
+        tokenizer.padding_side = "left" # use left-padding in generation
+
     data_collator = DataCollatorForSeq2Seq(
         tokenizer=tokenizer,
         label_pad_token_id=IGNORE_INDEX if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id