ros/LlamaFactory
1
0
Fork 0
mirror of https://github.com/hiyouga/LlamaFactory.git synced 2026-01-31 06:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix llava config

Browse Source 
Former-commit-id: b13d032325e45d401a9dbc64d4c73e308eff3288
This commit is contained in:
hiyouga
2024-05-12 00:02:49 +08:00
parent 436afcba57
commit 2bcd5b2b73
5 changed files with 15 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/llmtuner/model/utils/visual.py
View File

@@ -6,7 +6,7 @@ from ...extras.logging import get_logger
if TYPE_CHECKING:
from transformers import PreTrainedModel
from transformers import PretrainedConfig, PreTrainedModel
from ...hparams import ModelArguments
@@ -14,6 +14,11 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
def configure_hidden_size(config: "PretrainedConfig") -> None:
if getattr(config, "model_type", None) == "llava":
setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None))
def autocast_projector_dtype(
model: "PreTrainedModel", model_args: "ModelArguments", mm_projector_name: str = "multi_modal_projector"
) -> None:
@@ -22,7 +27,7 @@ def autocast_projector_dtype(
) -> "torch.Tensor":
return output.to(model_args.compute_dtype)
if hasattr(model, mm_projector_name):
if hasattr(model, mm_projector_name) and getattr(model.config, "quantization_method", None):
logger.info("Casting multimodal projector outputs in {}.".format(model_args.compute_dtype))
mm_projector: "torch.nn.Module" = getattr(model, mm_projector_name)
mm_projector.register_forward_hook(_mm_projector_forward_post_hook)