[infer] fix vllm args (#7235)

Former-commit-id: 999be5b4512890b8cf4f45874a77e35cf35626f5
This commit is contained in:
hoshi-hiyouga
2025-03-11 01:15:35 +08:00
committed by GitHub
parent 18968405d0
commit 522a3e8493
4 changed files with 32 additions and 26 deletions

View File

@@ -170,7 +170,7 @@ class VllmEngine(BaseEngine):
or 1.0, # repetition_penalty must > 0
temperature=temperature if temperature is not None else self.generating_args["temperature"],
top_p=(top_p if top_p is not None else self.generating_args["top_p"]) or 1.0, # top_p must > 0
top_k=top_k if top_k is not None else self.generating_args["top_k"],
top_k=(top_k if top_k is not None else self.generating_args["top_k"]) or -1, # top_k must > 0
stop=stop,
stop_token_ids=self.template.get_stop_token_ids(self.tokenizer),
max_tokens=max_tokens,