support unsloth

Former-commit-id: b857f00234b90b785d82ca7cdb29af3d948b1a7b

src/llmtuner/train/dpo/trainer.py

@@ -16,7 +16,7 @@ class CustomDPOTrainer(DPOTrainer):
     def __init__(
         self,
         beta: float,
-        loss_type: Literal["sigmoid", "hinge"],
+        loss_type: Literal["sigmoid", "hinge", "ipo", "kto"],
         ftx_gamma: float,
         model: Union["PreTrainedModel", torch.nn.Module],
         ref_model: Optional[Union["PreTrainedModel", torch.nn.Module]] = None,
@@ -28,16 +28,20 @@ class CustomDPOTrainer(DPOTrainer):
             if ref_model is not None:
                 disable_dropout_in_model(ref_model)
 
-        self.is_encoder_decoder = model.config.is_encoder_decoder
-        self.ref_model = ref_model
         self.use_dpo_data_collator = True # hack to avoid warning
         self.generate_during_eval = False # disable at evaluation
         self.label_pad_token_id = IGNORE_INDEX
         self.padding_value = 0
+        self.is_encoder_decoder = model.config.is_encoder_decoder
+        self.precompute_ref_log_probs = False
+        self._precomputed_train_ref_log_probs = False
+        self._precomputed_eval_ref_log_probs = False
+
+        self.ref_model = ref_model
         self.beta = beta
         self.label_smoothing = 0
-        self.ftx_gamma = ftx_gamma
         self.loss_type = loss_type
+        self.ftx_gamma = ftx_gamma
         self._stored_metrics = defaultdict(lambda: defaultdict(list))
 
         Trainer.__init__(self, model=model, **kwargs)
@@ -95,7 +99,7 @@ class CustomDPOTrainer(DPOTrainer):
         chosen_logits, rejected_logits = all_logits.split(batch_size, dim=0)
         return chosen_logps, rejected_logps, chosen_logits, rejected_logits
 
-    def get_batch_metrics(
+    def get_batch_loss_metrics(
         self,
         model: "PreTrainedModel",
         batch: Dict[str, torch.Tensor],

src/llmtuner/train/tuner.py

@@ -37,7 +37,7 @@ def run_exp(args: Optional[Dict[str, Any]] = None, callbacks: Optional[List["Tra
 def export_model(args: Optional[Dict[str, Any]] = None):
     model_args, _, finetuning_args, _ = get_infer_args(args)
 
-    if model_args.adapter_name_or_path is not None and finetuning_args.export_quantization_bit is not None:
+    if model_args.adapter_name_or_path is not None and model_args.export_quantization_bit is not None:
         raise ValueError("Please merge adapters before quantizing the model.")
 
     model, tokenizer = load_model_and_tokenizer(model_args, finetuning_args)
@@ -47,12 +47,12 @@ def export_model(args: Optional[Dict[str, Any]] = None):
 
     model.config.use_cache = True
     model = model.to("cpu")
-    model.save_pretrained(finetuning_args.export_dir, max_shard_size="{}GB".format(finetuning_args.export_size))
+    model.save_pretrained(model_args.export_dir, max_shard_size="{}GB".format(model_args.export_size))
 
     try:
         tokenizer.padding_side = "left" # restore padding side
         tokenizer.init_kwargs["padding_side"] = "left"
-        tokenizer.save_pretrained(finetuning_args.export_dir)
+        tokenizer.save_pretrained(model_args.export_dir)
     except:
         logger.warning("Cannot save tokenizer, please copy the files manually.")