ros/claude-code-router
1
0
Fork 0
mirror of https://github.com/musistudio/claude-code-router.git synced 2026-01-30 06:12:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix type error

Browse Source
This commit is contained in:
musistudio
2025-12-31 12:50:27 +08:00
parent b69ebf7ad1
commit f679670f92
2 changed files with 15 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
packages/core/src/tokenizer/tiktoken-tokenizer.ts
View File

@@ -1,8 +1,5 @@
import { get_encoding, Tiktoken } from "tiktoken"; import { get_encoding, Tiktoken, TiktokenEncoding } from "tiktoken";
import { import type { ITokenizer, TokenizeRequest } from "../types/tokenizer";
ITokenizer,
TokenizeRequest,
} from "../types/tokenizer";
/** /**
* Tiktoken-based tokenizer implementation * Tiktoken-based tokenizer implementation
@@ -12,10 +9,8 @@ export class TiktokenTokenizer implements ITokenizer {
readonly type = "tiktoken"; readonly type = "tiktoken";
readonly name: string; readonly name: string;
private encoding?: Tiktoken; private encoding?: Tiktoken;
private encodingName: string;
constructor(encodingName: string = "cl100k_base") { constructor(encodingName: TiktokenEncoding = "cl100k_base") {
this.encodingName = encodingName;
this.name = `tiktoken-${encodingName}`; this.name = `tiktoken-${encodingName}`;
try { try {
this.encoding = get_encoding(encodingName); this.encoding = get_encoding(encodingName);
@@ -32,7 +27,8 @@ export class TiktokenTokenizer implements ITokenizer {
} }
async countTokens(request: TokenizeRequest): Promise<number> { async countTokens(request: TokenizeRequest): Promise<number> {
if (!this.encoding) { const encoding = this.encoding;
if (!encoding) {
throw new Error("Encoding not initialized"); throw new Error("Encoding not initialized");
} }
@@ -43,13 +39,13 @@ export class TiktokenTokenizer implements ITokenizer {
if (Array.isArray(messages)) { if (Array.isArray(messages)) {
messages.forEach((message) => { messages.forEach((message) => {
if (typeof message.content === "string") { if (typeof message.content === "string") {
tokenCount += this.encoding!.encode(message.content).length; tokenCount += encoding.encode(message.content).length;
} else if (Array.isArray(message.content)) { } else if (Array.isArray(message.content)) {
message.content.forEach((contentPart: any) => { message.content.forEach((contentPart: any) => {
if (contentPart.type === "text") { if (contentPart.type === "text") {
tokenCount += this.encoding!.encode(contentPart.text).length; tokenCount += encoding.encode(contentPart.text).length;
} else if (contentPart.type === "tool_use") { } else if (contentPart.type === "tool_use") {
tokenCount += this.encoding!.encode( tokenCount += encoding.encode(
JSON.stringify(contentPart.input) JSON.stringify(contentPart.input)
).length; ).length;
} else if (contentPart.type === "tool_result") { } else if (contentPart.type === "tool_result") {
@@ -57,7 +53,7 @@ export class TiktokenTokenizer implements ITokenizer {
typeof contentPart.content === "string" typeof contentPart.content === "string"
? contentPart.content ? contentPart.content
: JSON.stringify(contentPart.content); : JSON.stringify(contentPart.content);
tokenCount += this.encoding!.encode(content).length; tokenCount += encoding.encode(content).length;
} }
}); });
} }
@@ -66,15 +62,15 @@ export class TiktokenTokenizer implements ITokenizer {
// Count system // Count system
if (typeof system === "string") { if (typeof system === "string") {
tokenCount += this.encoding.encode(system).length; tokenCount += encoding.encode(system).length;
} else if (Array.isArray(system)) { } else if (Array.isArray(system)) {
system.forEach((item: any) => { system.forEach((item: any) => {
if (item.type !== "text") return; if (item.type !== "text") return;
if (typeof item.text === "string") { if (typeof item.text === "string") {
tokenCount += this.encoding.encode(item.text).length; tokenCount += encoding.encode(item.text).length;
} else if (Array.isArray(item.text)) { } else if (Array.isArray(item.text)) {
item.text.forEach((textPart: any) => { item.text.forEach((textPart: any) => {
tokenCount += this.encoding.encode(textPart || "").length; tokenCount += encoding.encode(textPart || "").length;
}); });
} }
}); });
@@ -84,12 +80,12 @@ export class TiktokenTokenizer implements ITokenizer {
if (tools) { if (tools) {
tools.forEach((tool: any) => { tools.forEach((tool: any) => {
if (tool.description) { if (tool.description) {
tokenCount += this.encoding.encode( tokenCount += encoding.encode(
tool.name + tool.description tool.name + tool.description
).length; ).length;
} }
if (tool.input_schema) { if (tool.input_schema) {
tokenCount += this.encoding.encode( tokenCount += encoding.encode(
JSON.stringify(tool.input_schema) JSON.stringify(tool.input_schema)
).length; ).length;
} }

2
packages/core/tsconfig.json
View File

@@ -2,7 +2,7 @@
"compilerOptions": { "compilerOptions": {
"target": "ES2022", "target": "ES2022",
"module": "CommonJS", "module": "CommonJS",
"lib": ["ES2022"], "lib": ["ES2022", "DOM"],
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",
"strict": true, "strict": true,