ros/LLaMA-Factory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[model] update kt code (#9406)

Browse Source
This commit is contained in:
Yaowei Zheng
2025-11-05 15:27:22 +08:00
committed by GitHub
parent 56f45e826f
commit eaf963f67f
28 changed files with 108 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/kt_optimize_rules/DeepSeek-V3-Chat-amx.yaml
View File

@@ -7,7 +7,7 @@
prefill_device: "cuda"
- match:
name: "^lm_head$" # regular expression
name: "^lm_head$" # regular expression
class: torch.nn.Linear # only match modules matching name and class simultaneously
replace:
class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types
@@ -18,7 +18,7 @@
prefill_op: "KLinearTorch"
- match:
name: "^model\\.layers\\.(?!.*self_attn\\.kv_b_proj).*$" # regular expression
name: "^model\\.layers\\.(?!.*self_attn\\.kv_b_proj).*$" # regular expression
class: torch.nn.Linear # only match modules matching name and class simultaneously
replace:
class: ktransformers.operators.linear.KTransformersLinear # optimized Kernel on quantized data types
@@ -74,4 +74,4 @@
class: "default"
kwargs:
generate_device: "cpu"
prefill_device: "cpu"
prefill_device: "cpu"