ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix chatglm2 tokenizer

Browse Source 
Former-commit-id: 1ab60b4a93fa1be5dfe6ffbd4deb64c0f9d9b431
This commit is contained in:
hiyouga
2023-09-09 13:50:29 +08:00
parent 9f83e93839
commit 50e93392dd
4 changed files with 17 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tests/llamafy_baichuan2.py
View File

@@ -1,7 +1,6 @@
# coding=utf-8
# Converts the Baichuan2-7B model in the same format as LLaMA2-7B.
# Usage: python llamafy_baichuan2.py --baichuan2_json baichuan2.index.json --llama2_json llama2.index.json
# --input_dir baichuan2_original --output_dir baichuan2_llamafied
# Usage: python llamafy_baichuan2.py --llama2_json llama2.index.json --input_dir input --output_dir output
# Inspired by: https://huggingface.co/fireballoon/baichuan-llama-7b/blob/main/convert_baichuan_to_llama.py
# Converted model: https://huggingface.co/hiyouga/Baichuan2-7B-Base-LLaMAfied
@@ -17,20 +16,20 @@ SHARD_B = "pytorch_model-00002-of-00002.bin"
def llamafy_baichuan2(
baichuan2_json: str,
llama2_json: str,
input_dir: str,
output_dir: str
):
weight_shard_a = torch.load(os.path.join(input_dir, SHARD_A), map_location="cpu")
weight_shard_b = torch.load(os.path.join(input_dir, SHARD_B), map_location="cpu")
baichuan2_state_dict = OrderedDict()
baichuan2_state_dict.update(weight_shard_a)
baichuan2_state_dict.update(weight_shard_b)
for filepath in os.listdir(input_dir):
if os.path.isfile(os.path.join(input_dir, filepath)) and filepath.endswith(".bin"):
shard_weight = torch.load(os.path.join(input_dir, filepath), map_location="cpu")
baichuan2_state_dict.update(shard_weight)
llama2_state_dict = OrderedDict()
total_size = 0
for key, value in baichuan2_state_dict.items():
total_size += 2 * value.numel() # half precision
if "W_pack" in key:
llama2_state_dict[key.replace("W_pack", "q_proj")] = value[:4096, :]
llama2_state_dict[key.replace("W_pack", "k_proj")] = value[4096:2*4096, :]
@@ -40,13 +39,11 @@ def llamafy_baichuan2(
else:
llama2_state_dict[key] = value
with open(os.path.join(input_dir, baichuan2_json), "r", encoding="utf-8") as f:
baichuan2_index = json.load(f)
with open(os.path.join(input_dir, llama2_json), "r", encoding="utf-8") as f:
llama2_index = json.load(f)
merged_index = OrderedDict()
merged_index["metadata"] = baichuan2_index["metadata"]
merged_index["metadata"] = {"total_size": total_size}
merged_index["weight_map"] = llama2_index["weight_map"]
state_dict_a, state_dict_b = OrderedDict(), OrderedDict()
@@ -60,7 +57,7 @@ def llamafy_baichuan2(
torch.save(state_dict_a, os.path.join(output_dir, SHARD_A))
torch.save(state_dict_b, os.path.join(output_dir, SHARD_B))
with open(os.path.join(output_dir, "pytorch_model.bin.index.json"), "w", encoding="utf-8") as f:
json.dump(merged_index, f)
json.dump(merged_index, f, indent=2)
print("Completed!")