support Qwen-7B, fix InternLM-7B inference

Former-commit-id: 25d2ca29ecb70cbfd5206333c667042a0c4d2e5a
2023-08-03 15:53:32 +08:00
parent da08fa7c63
commit 2e19afedb8
8 changed files with 89 additions and 25 deletions
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,7 @@ accelerate>=0.21.0
 peft>=0.4.0
 trl>=0.4.7
 sentencepiece
+tiktoken
 jieba
 rouge-chinese
 nltk